diff --git a/README.rst b/README.rst index 6e01458e97..3b469375db 100644 --- a/README.rst +++ b/README.rst @@ -47,36 +47,81 @@ it's often difficult to work with. PUDL takes the original spreadsheets, CSV fil and databases and turns them into a unified resource. This allows users to spend more time on novel analysis and less time on data preparation. -Who is PUDL for? ----------------- - The project is focused on serving researchers, activists, journalists, policy makers, -and small businesses that might not otherwise be able to afford access to this data from -commercial sources and who may not have the time or expertise to do all the data -processing themselves from scratch. +and small businesses that might not otherwise be able to afford access to this data +from commercial sources and who may not have the time or expertise to do all the +data processing themselves from scratch. We want to make this data accessible and easy to work with for as wide an audience as -possible: anyone from a grassroots youth climate organizers working with Google sheets -to university researchers with access to scalable cloud computing resources and everyone -in between! +possible: anyone from a grassroots youth climate organizers working with Google +sheets to university researchers with access to scalable cloud computing +resources and everyone in between! + +PUDL is comprised of three core components: + +- **Raw Data Archives** + + - PUDL `archives `__ + all the raw data inputs on `Zenodo `__ + to ensure perminant, versioned access to the data. In the event that an agency + changes how they publish data or deletes old files, the ETL will still have access + to the original inputs. Each of the data inputs may have several different versions + archived, and all are assigned a unique DOI and made available through the REST API. + You can read more about the Raw Data Archives in the + `docs `__. +- **ETL Pipeline** + + - The ETL pipeline (this repo) ingests the raw archives, cleans them, + integrates them, and outputs them to a series of tables stored in SQLite Databases, + Parquet files, and pickle files (the Data Warehouse). Each release of the PUDL + Python package is embedded with a set of of DOIs to indicate which version of the + raw inputs it is meant to process. This process helps ensure that the ETL and it's + outputs are replicable. You can read more about the ETL in the + `docs `__. +- **Data Warehouse** + + - The outputs from the ETL, sometimes called "PUDL outputs", + are stored in a data warehouse as a collection of SQLite and Parquet files so that + users can access the data without having to run any code. Learn more about how to + access the data `here `__. What data is available? ----------------------- PUDL currently integrates data from: -* `EIA Form 860 `__: 2001 - 2022 -* `EIA Form 860m `__: 2023-06 -* `EIA Form 861 `__: 2001 - 2022 -* `EIA Form 923 `__: 2001 - 2023-08 -* `EPA Continuous Emissions Monitoring System (CEMS) `__: 1995 - 2022 -* `FERC Form 1 `__: 1994-2021 -* `FERC Form 714 `__: 2006-2020 -* `US Census Demographic Profile 1 Geodatabase `__: 2010 +* **EIA Form 860**: 2001-2022 + - `Source Docs `__ + - `PUDL Docs `__ +* **EIA Form 860m**: 2023-06 + - `Source Docs `__ +* **EIA Form 861**: 2001-2022 + - `Source Docs `__ + - `PUDL Docs `__ +* **EIA Form 923**: 2001-2022 + - `Source Docs `__ + - `PUDL Docs `__ +* **EPA Continuous Emissions Monitoring System (CEMS)**: 1995-2022 + - `Source Docs `__ + - `PUDL Docs `__ +* **FERC Form 1**: 1994-2021 + - `Source Docs `__ + - `PUDL Docs `__ +* **FERC Form 714**: 2006-2020 + - `Source Docs `__ + - `PUDL Docs `__ +* **FERC Form 2**: 2021 (raw only) + - `Source Docs `__ +* **FERC Form 6**: 2021 (raw only) + - `Source Docs `__ +* **FERC Form 60**: 2021 (raw only) + - `Source Docs `__ +* **US Census Demographic Profile 1 Geodatabase**: 2010 + - `Source Docs `__ Thanks to support from the `Alfred P. Sloan Foundation Energy & Environment Program `__, from -2021 to 2024 we will be integrating the following data as well: +2021 to 2024 we will be cleaning and integrating the following data as well: * `EIA Form 176 `__ (The Annual Report of Natural Gas Supply and Disposition) diff --git a/devtools/debug-eia-etl.ipynb b/devtools/debug-eia-etl.ipynb index a1703bbc7d..347acd9624 100644 --- a/devtools/debug-eia-etl.ipynb +++ b/devtools/debug-eia-etl.ipynb @@ -263,7 +263,7 @@ "outputs": [], "source": [ "%%time\n", - "asset_key = \"fuel_receipts_costs_eia923\"\n", + "asset_key = \"core_eia923__monthly_fuel_receipts_costs\"\n", "df = defs.load_asset_value(AssetKey(asset_key))\n", "\n", "df.head()" diff --git a/devtools/debug-ferc1-etl.ipynb b/devtools/debug-ferc1-etl.ipynb index 19caff6102..1b179ea812 100644 --- a/devtools/debug-ferc1-etl.ipynb +++ b/devtools/debug-ferc1-etl.ipynb @@ -99,7 +99,7 @@ }, "outputs": [], "source": [ - "ferc1_xbrl_raw_dfs[\"fuel_ferc1\"][\"duration\"].report_year" + "ferc1_xbrl_raw_dfs[\"core_ferc1__yearly_steam_plants_fuel_sched402\"][\"duration\"].report_year" ] }, { @@ -206,7 +206,7 @@ "metadata": {}, "outputs": [], "source": [ - "table_name = \"other_regulatory_liabilities_ferc1\"\n", + "table_name = \"core_ferc1__yearly_other_regulatory_liabilities_sched278\"\n", "TRANSFORMER = transformers[table_name] # add a table here" ] }, @@ -326,8 +326,8 @@ "source": [ "transformed_tables = {}\n", "for table_name, transformer in transformers.items():\n", - " if table_name == \"plants_steam_ferc1\":\n", - " # plants_steam_ferc1 is a special case. It depends on the transformed fuel_ferc1 table.\n", + " if table_name == \"core_ferc1__yearly_steam_plants_sched402\":\n", + " # core_ferc1__yearly_steam_plants_sched402 is a special case. It depends on the transformed core_ferc1__yearly_steam_plants_fuel_sched402 table.\n", " continue\n", " transformed_tables[transformer.table_id.value] = transformer.transform(\n", " raw_dbf=ferc1_dbf_raw_dfs[transformer.table_id.value],\n", @@ -345,13 +345,13 @@ }, "outputs": [], "source": [ - "# Handle special case for \"plants_steam_ferc1\"\n", - "transformer = transformers[\"plants_steam_ferc1\"]\n", + "# Handle special case for \"core_ferc1__yearly_steam_plants_sched402\"\n", + "transformer = transformers[\"core_ferc1__yearly_steam_plants_sched402\"]\n", "transformed_tables[transformer.table_id.value] = transformer.transform(\n", " raw_dbf=ferc1_dbf_raw_dfs[transformer.table_id.value],\n", " raw_xbrl_instant=ferc1_xbrl_raw_dfs[transformer.table_id.value][\"instant\"],\n", " raw_xbrl_duration=ferc1_xbrl_raw_dfs[transformer.table_id.value][\"duration\"],\n", - " transformed_fuel=transformed_tables[\"fuel_ferc1\"],\n", + " transformed_fuel=transformed_tables[\"core_ferc1__yearly_steam_plants_fuel_sched402\"],\n", ")" ] } @@ -372,7 +372,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/devtools/ferc1-eia-glue/training_data/create_FERC1-EIA_manual_mapping_spreadsheets.ipynb b/devtools/ferc1-eia-glue/training_data/create_FERC1-EIA_manual_mapping_spreadsheets.ipynb index c45e2e008f..c73ff0e8d9 100644 --- a/devtools/ferc1-eia-glue/training_data/create_FERC1-EIA_manual_mapping_spreadsheets.ipynb +++ b/devtools/ferc1-eia-glue/training_data/create_FERC1-EIA_manual_mapping_spreadsheets.ipynb @@ -43,7 +43,7 @@ "# Local libraries\n", "import pudl\n", "from pudl.workspace.setup import PudlPaths\n", - "from pudl.analysis.ferc1_eia_train import *" + "from pudl.analysis.eia_ferc1_train import *" ] }, { diff --git a/devtools/ferc1-eia-glue/training_data/validate_and_integrate_FERC1-EIA_manually_mapped_records.ipynb b/devtools/ferc1-eia-glue/training_data/validate_and_integrate_FERC1-EIA_manually_mapped_records.ipynb index 22149567ec..72a5b63e01 100644 --- a/devtools/ferc1-eia-glue/training_data/validate_and_integrate_FERC1-EIA_manually_mapped_records.ipynb +++ b/devtools/ferc1-eia-glue/training_data/validate_and_integrate_FERC1-EIA_manually_mapped_records.ipynb @@ -44,7 +44,7 @@ "# Local libraries\n", "import pudl\n", "from pudl.workspace.setup import PudlPaths\n", - "from pudl.analysis.ferc1_eia_train import *" + "from pudl.analysis.eia_ferc1_train import *" ] }, { @@ -188,7 +188,7 @@ "outputs": [], "source": [ "current_training_df = pd.read_csv(\n", - " importlib.resources.files(\"pudl.package_data.glue\").joinpath(\"ferc1_eia_train.csv\")\n", + " importlib.resources.files(\"pudl.package_data.glue\").joinpath(\"eia_ferc1_train.csv\")\n", ")\n", "path_to_overrides = \"./add_to_training/\"\n", "override_files = [\n", @@ -326,10 +326,10 @@ "# Get paths to CSVs.\n", "from importlib import resources\n", "one_to_many = path_to_one_to_many=resources.files(\"pudl.package_data.glue\").joinpath(\n", - " \"ferc1_eia_one_to_many.csv\",\n", + " \"eia_ferc1_one_to_many.csv\",\n", " )\n", "nulls = path_to_one_to_many=resources.files(\"pudl.package_data.glue\").joinpath(\n", - " \"ferc1_eia_null.csv\",\n", + " \"eia_ferc1_null.csv\",\n", " )" ] }, diff --git a/devtools/inspect-assets.ipynb b/devtools/inspect-assets.ipynb index b5595f75b7..f09be07c94 100644 --- a/devtools/inspect-assets.ipynb +++ b/devtools/inspect-assets.ipynb @@ -50,10 +50,61 @@ "\n", "from pudl.etl import defs\n", "\n", - "asset_key = \"raw_generator_existing_eia860\"\n", + "asset_key = \"exploded_balance_sheet_assets_ferc1\"\n", "df = defs.load_asset_value(AssetKey(asset_key))\n", "\n", - "df.head()" + "#df[df.row_type_xbrl == \"correction\"].xbrl_factoid.value_counts()\n", + "#df[(df.xbrl_factoid.isin([\"operation_expense\", \"maintenance_expense\"]))&(df.rel_diff.notnull())&(df.rel_diff!=0)].sort_values(['utility_id_ferc1', 'report_year', 'xbrl_factoid', 'rel_diff']).head(50)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2d99594", + "metadata": {}, + "outputs": [], + "source": [ + "df[(df.xbrl_factoid==\"accumulated_depreciation\")&(df.plant_status==\"in_service\")&(df.plant_function==\"total\")]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "467111b1", + "metadata": {}, + "outputs": [], + "source": [ + "df[df.xbrl_factoid.isin(factoids)&(df.utility_id_ferc1==9)&(df.report_year==1998)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6f7427a", + "metadata": {}, + "outputs": [], + "source": [ + "factoids = ['distribution_maintenance_expense_electric',\n", + " 'hydraulic_power_generation_maintenance_expense',\n", + " 'maintenance_of_general_plant',\n", + " 'nuclear_power_generation_maintenance_expense',\n", + " 'other_power_generation_maintenance_expense',\n", + " 'regional_market_maintenance_expense',\n", + " 'steam_power_generation_maintenance_expense',\n", + " 'transmission_maintenance_expense_electric']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "951b718d", + "metadata": {}, + "outputs": [], + "source": [ + "asset_key = \"calculation_components_xbrl_ferc1\"\n", + "calcs = defs.load_asset_value(AssetKey(asset_key))\n", + "\n", + "calcs[(calcs.xbrl_factoid_parent == \"accumulated_depreciation\")].head(50)" ] }, { @@ -77,10 +128,24 @@ "\n", "from pudl.etl import defs\n", "\n", - "asset_key = \"fuel_receipts_costs_eia923\"\n", + "asset_key = \"emissions_unit_ids_epacems\"\n", "df = defs.load_asset_value(AssetKey(asset_key))\n", "\n", - "df.head()" + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f0d118b", + "metadata": {}, + "outputs": [], + "source": [ + "from pudl.output.epacems import epacems\n", + "\n", + "test_epacems = epacems(states = [\"ID\"], years = [2022])\n", + "\n", + "test_epacems[test_epacems.operating_datetime_utc>=\"2022-01-04\"].head(40)" ] } ], @@ -100,7 +165,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/devtools/python-output-table-conversion-debug.ipynb b/devtools/python-output-table-conversion-debug.ipynb index 94748df138..538c212997 100644 --- a/devtools/python-output-table-conversion-debug.ipynb +++ b/devtools/python-output-table-conversion-debug.ipynb @@ -48,14 +48,14 @@ "\n", "You can create an asset by creating a new function and adding the `@asset` decorator. For now, the only attribute you should add to the decorator is the `compute_type = \"Python\"`. All this does is add a cute tag to the asset in the dag to let people know how the asset is being processed.\n", "\n", - "Next you'll want to figure out what tables the output table depends on. Read through the old output function to see which normalized tables or output functions are being used as inputs to the joins and imputations. Once you have the input table names, add them to the asset function parameters. For example, the `utilities_eia860()` function merges `utilities_entity_eia`, `utilities_eia860`, and `utilities_eia` tables together so the asset would look like this:\n", + "Next you'll want to figure out what tables the output table depends on. Read through the old output function to see which normalized tables or output functions are being used as inputs to the joins and imputations. Once you have the input table names, add them to the asset function parameters. For example, the `utilities_eia860()` function merges `core_eia__entity_utilities`, `core_eia860__scd_utilities`, and `core_pudl__assn_eia_pudl_utilities` tables together so the asset would look like this:\n", "\n", "```python\n", "@asset(compute_kind=\"Python\")\n", "def denorm_utilities_eia860(\n", - " utilities_entity_eia: pd.DataFrame,\n", - " utilities_eia860: pd.DataFrame,\n", - " utilities_eia: pd.DataFrame,\n", + " core_eia__entity_utilities: pd.DataFrame,\n", + " core_eia860__scd_utilities: pd.DataFrame,\n", + " core_pudl__assn_eia_pudl_utilities: pd.DataFrame,\n", "):\n", " ... # joining logic\n", " return joined_df\n", @@ -108,9 +108,9 @@ "```python\n", "@asset(io_manager_key=\"pudl_sqlite_io_manager\", compute_kind=\"Python\")\n", "def denorm_utilities_eia860(\n", - " utilities_entity_eia: pd.DataFrame,\n", - " utilities_eia860: pd.DataFrame,\n", - " utilities_eia: pd.DataFrame,\n", + " core_eia__entity_utilities: pd.DataFrame,\n", + " core_eia860__scd_utilities: pd.DataFrame,\n", + " core_pudl__assn_eia_pudl_utilities: pd.DataFrame,\n", "):\n", " ... # joining logic\n", " return joined_df\n", diff --git a/docs/data_access.rst b/docs/data_access.rst index 3fb5655b74..e4907e0533 100644 --- a/docs/data_access.rst +++ b/docs/data_access.rst @@ -2,12 +2,17 @@ Data Access ======================================================================================= -We publish the :doc:`PUDL pipeline ` outputs in several ways to serve +We publish the PUDL pipeline outputs in several ways to serve different users and use cases. We're always trying to increase accessibility of the PUDL data, so if you have a suggestion please `open a GitHub issue `__. If you have a question you can `create a GitHub discussion `__. +PUDL's primary data output is the ``pudl.sqlite`` database. We recommend working with +tables with the ``out_`` prefix, as these tables contain the most complete and easiest +to work with data. For more information about the different types +of tables, read through :ref:`PUDL's naming conventions `. + .. _access-modes: --------------------------------------------------------------------------------------- diff --git a/docs/dev/data_guidelines.rst b/docs/dev/data_guidelines.rst index 53bcb1c18a..dfbd40c0db 100644 --- a/docs/dev/data_guidelines.rst +++ b/docs/dev/data_guidelines.rst @@ -64,6 +64,8 @@ Examples of Unacceptable Changes fuel heat content and net electricity generation. The heat rate would be a derived value and not part of the original data. +.. _tidy-data: + ------------------------------------------------------------------------------- Make Tidy Data ------------------------------------------------------------------------------- @@ -117,24 +119,6 @@ that M/Mega is a million in SI. And a `BTU energy required to raise the temperature of one an *avoirdupois pound* of water by 1 degree *Farenheit*?! What century even is this?). -------------------------------------------------------------------------------- -Silo the ETL Process -------------------------------------------------------------------------------- -It should be possible to run the ETL process on each data source independently -and with any combination of data sources included. This allows users to include -only the data need. In some cases, like the :doc:`EIA 860 -<../data_sources/eia860>` and :doc:`EIA 923 <../data_sources/eia923>` data, two -data sources may be so intertwined that keeping them separate doesn't really -make sense. This should be the exception, however, not the rule. - -------------------------------------------------------------------------------- -Separate Data from Glue -------------------------------------------------------------------------------- -The glue that relates different data sources to each other should be applied -after or alongside the ETL process and not as a mandatory part of ETL. This -makes it easy to pull individual data sources in and work with them even when -the glue isn't working or doesn't yet exist. - ------------------------------------------------------------------------------- Partition Big Data ------------------------------------------------------------------------------- @@ -146,35 +130,6 @@ them to pull in only certain years, certain states, or other sensible partitions data so that they don’t run out of memory or disk space or have to wait hours while data they don't need is being processed. -------------------------------------------------------------------------------- -Naming Conventions -------------------------------------------------------------------------------- - *There are only two hard problems in computer science: caching, - naming things, and off-by-one errors.* - -Use Consistent Names -^^^^^^^^^^^^^^^^^^^^ -If two columns in different tables record the same quantity in the same units, -give them the same name. That way if they end up in the same dataframe for -comparison it's easy to automatically rename them with suffixes indicating -where they came from. For example, net electricity generation is reported to -both :doc:`FERC Form 1 <../data_sources/ferc1>` and :doc:`EIA 923 -<../data_sources/eia923>`, so we've named columns ``net_generation_mwh`` in -each of those data sources. Similarly, give non-comparable quantities reported -in different data sources **different** column names. This helps make it clear -that the quantities are actually different. - -Follow Existing Conventions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We are trying to use consistent naming conventions for the data tables, -columns, data sources, and functions. Generally speaking PUDL is a collection -of subpackages organized by purpose (extract, transform, load, analysis, -output, datastore…), containing a module for each data source. Each data source -has a short name that is used everywhere throughout the project and is composed of -the reporting agency and the form number or another identifying abbreviation: -``ferc1``, ``epacems``, ``eia923``, ``eia861``, etc. See the :doc:`naming -conventions ` document for more details. - ------------------------------------------------------------------------------- Complete, Continuous Time Series ------------------------------------------------------------------------------- diff --git a/docs/dev/dev_setup.rst b/docs/dev/dev_setup.rst index de7ec65b6a..f27abdd470 100644 --- a/docs/dev/dev_setup.rst +++ b/docs/dev/dev_setup.rst @@ -291,7 +291,7 @@ directory: $ set -Ux PUDL_OUTPUT /absolute/path/to/pudl_output # fish The path stored in ``PUDL_OUTPUT`` contains all ETL outputs like ``pudl.sqlite`` and -``hourly_emissions_epacems.parquet``. +``core_epacems__hourly_emissions.parquet``. .. warning:: diff --git a/docs/dev/existing_data_updates.rst b/docs/dev/existing_data_updates.rst index e25152a282..696feb3e47 100644 --- a/docs/dev/existing_data_updates.rst +++ b/docs/dev/existing_data_updates.rst @@ -344,10 +344,10 @@ A. FERC 1 & EIA Plants & Utilities B. Missing EIA Plant Locations from CEMS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ **6.B.1)** If there are any plants that appear in the EPA CEMS dataset that do not -appear in the ``plants_entity_eia`` table, or that are missing latitude and longitude -values, you'll get a warning when you try and materialize the ``epacamd`` asset group in -Dagster. You'll need to manually compile the missing information and add it to -``src/pudl/package_data/epacems/additional_epacems_plants.csv`` to enable accurate +appear in the ``core_eia__entity_plants`` table, or that are missing latitude and +longitude values, you'll get a warning when you try and materialize the ``core_epacamd`` +asset group in Dagster. You'll need to manually compile the missing information and add +it to ``src/pudl/package_data/epacems/additional_epacems_plants.csv`` to enable accurate adjustment of the EPA CEMS timestamps to UTC. Using the Plant ID from the warning, look up the plant coordinates in the `EPA FACT API `__. diff --git a/docs/dev/naming_conventions.rst b/docs/dev/naming_conventions.rst index 1b109dce9e..4993fb77ec 100644 --- a/docs/dev/naming_conventions.rst +++ b/docs/dev/naming_conventions.rst @@ -1,6 +1,194 @@ =============================================================================== Naming Conventions =============================================================================== + *There are only two hard problems in computer science: caching, + naming things, and off-by-one errors.* + +We try to use consistent naming conventions for the data tables, data assets, +columns, data sources, and functions. + +.. _asset-naming: + +Asset Naming Conventions +--------------------------------------------------- + +PUDL's data processing is divided into three layers of Dagster assets: Raw, Core +and Output. Dagster assets are the core unit of computation in PUDL. The outputs +of assets can be persisted to any type of storage though PUDL outputs are typically +tables in a SQLite database, parquet files or pickle files (read more about this here: +:doc:`../index`). The asset name is used for the table or parquet file name. Asset +names should generally follow this naming convention: + +.. code-block:: + + {layer}_{source}__{asset_type}_{asset_name} + +* ``layer`` is the processing layer of the asset. Acceptable values are: + ``raw``, ``core`` and ``out``. ``layer`` is required for all assets in all layers. +* ``source`` is an abbreviation of the original source of the data. For example, + ``eia860``, ``ferc1`` and ``epacems``. +* ``asset_type`` describes how the asset in modeled. +* ``asset_name`` should describe the entity, categorical code type, or measurement of + the asset. Note: FERC Form 1 assets typically include the schedule number in the + ``asset_name`` so users and contributors know which schedule the cleaned asset + refers to. + +Raw layer +^^^^^^^^^ +This layer contains assets that extract data from spreadsheets and databases +and are persisted as pickle files. + +Naming convention: ``raw_{source}__{asset_name}`` + +* ``asset_name`` is typically copied from the source data. +* ``asset_type`` is not included in this layer because the data modeling does not + yet conform to PUDL standards. Raw assets are typically just copies of the + source data. + +Core layer +^^^^^^^^^^ +This layer contains assets that typically break denormalized raw assets into +well-modeled tables that serve as building blocks for downstream wide tables +and analyses. Well-modeled means tables in the database have logical +primary keys, foreign keys, datatypes and generally follow +:ref:`Tidy Data standards `. Assets in this layer create +consistent categorical variables, decuplicate and impute data. +These assets are typically stored in parquet files or tables in a database. + +Naming convention: ``core_{source}__{asset_type}_{asset_name}`` + +* ``source`` is sometimes ``pudl``. This means the asset + is a derived connection the contributors of PUDL created to connect multiple + datasets via manual or machine learning methods. + +* ``asset_type`` describes how the asset is modeled and its role in PUDL’s + collection of core assets. There are a handful of table types in this layer: + + * ``assn``: Association assets provide connections between entities. They should + follow this naming convention: + + ``{layer}_{source of association asset}__assn_{datasets being linked}_{entity + being linked}`` + + Association assets can be manually compiled or extracted from data sources. If + the asset associates data from two sources, the source names should be included + in the ``asset_name`` in alphabetical order. Examples: + + * ``core_pudl__assn_plants_eia`` associates EIA Plant IDs and manually assigned + PUDL Plant IDs. + * ``core_epa__assn_epacamd_eia`` associates EPA units with EIA plants, boilers, + and generators. + * ``codes``: Code tables contain more verbose descriptions of categorical codes + typically manually compiled from source data dictionaries. Examples: + + * ``core_eia__codes_averaging_periods`` + * ``core_eia__codes_balancing_authorities`` + * ``entity``: Entity tables contain static information about entities. For example, + the state a plant is located in or the plant a boiler is a part of. Examples: + + * ``core_eia__entity_boilers`` + * ``core_eia923__entity_coalmine``. + * ``scd``: Slowly changing dimension tables describe attributes of entities that + rarely change. For example, the ownership or the capacity of a plant. Examples: + + * ``core_eia860__scd_generators`` + * ``core_eia860__scd_plants``. + * ``yearly/monthly/hourly``: Time series tables contain attributes about entities + that are expected to change for each reported timestamp. Time series tables + typically contain measurements of processes like net generation or co2 emissions. + Examples: + + * ``core_ferc714__hourly_demand_pa``, + * ``core_ferc1__yearly_plant_in_service``. + +Output layer +^^^^^^^^^^^^ +This layer uses assets in the Core layer to construct wide and complete tables +suitable for users to perform analysis on. This layer can contain intermediate +tables that bridge the core and user-facing tables. + +Naming convention: ``out_{source}__{asset_type}_{asset_name}`` + +* ``source`` is optional in this layer because there can be assets that join data from + multiple sources. +* ``asset_type`` is also optional. It will likely describe the frequency at which + the data is reported (annual/monthly/hourly). + +Intermediate Assets +^^^^^^^^^^^^^^^^^^^ +Intermediate assets are logical steps towards a final well-modeled core or +user-facing output asset. These assets are not intended to be persisted in the +database or accessible to the user. These assets are denoted by a preceding +underscore, like a private python method. For example, the intermediate asset +``_core_eia860__plants`` is a logical step towards the +``core_eia860__entity_plants`` and ``core_eia860__scd_plants`` assets. +``_core_eia860__plants`` does some basic cleaning of the ``raw_eia860__plant`` +asset but still contains duplicate plant entities. The computation intensive +harvesting process deduplicates ``_core_eia860__plants`` and outputs the +``core_eia860__entity_plants`` and ``core_eia860__scd_plants`` assets which +follow Tiny Data standards. + +Limit the number of intermediate assets to avoid an extremely +cluttered DAG. It is appropriate to create an intermediate asset when: + + * there is a short and long running portion of a process. It is convenient to separate + the long and short-running processing portions into separate assets so debugging the + short-running process doesn’t take forever. + * there is a logical step in a process that is frequently inspected for debugging. For + example, the pre harvest assets in the ``_core_eia860`` and ``_core_eia923`` groups + are frequently inspected when new years of data are added. + + +Columns and Field Names +----------------------- +If two columns in different tables record the same quantity in the same units, +give them the same name. That way if they end up in the same dataframe for +comparison it's easy to automatically rename them with suffixes indicating +where they came from. For example, net electricity generation is reported to +both :doc:`FERC Form 1 <../data_sources/ferc1>` and +:doc:`EIA 923<../data_sources/eia923>`, so we've named columns ``net_generation_mwh`` +in each of those data sources. Similarly, give non-comparable quantities reported in +different data sources **different** column names. This helps make it clear that the +quantities are actually different. + +* ``total`` should come at the beginning of the name (e.g. + ``total_expns_production``) +* Identifiers should be structured ``type`` + ``_id_`` + ``source`` where + ``source`` is the agency or organization that has assigned the ID. (e.g. + ``plant_id_eia``) +* The data source or label (e.g. ``plant_id_pudl``) should follow the thing it + is describing +* Append units to field names where applicable (e.g. + ``net_generation_mwh``). This includes "per unit" signifiers (e.g. ``_pct`` + for percent, ``_ppm`` for parts per million, or a generic ``_per_unit`` when + the type of unit varies, as in columns containing a heterogeneous collection + of fuels) +* Financial values are assumed to be in nominal US dollars (I.e., the suffix + _usd is implied.)If they are not reported in USD, convert them to USD. If + they must be kept in their original form for some reason, append a suffix + that lets the user know they are not USD. +* ``_id`` indicates the field contains a usually numerical reference to + another table, which will not be intelligible without looking up the value in + that other table. +* The suffix ``_code`` indicates the field contains a short abbreviation from + a well defined list of values, that probably needs to be looked up if you + want to understand what it means. +* The suffix ``_type`` (e.g. ``fuel_type``) indicates a human readable category + from a well defined list of values. Whenever possible we try to use these + longer descriptive names rather than codes. +* ``_name`` indicates a longer human readable name, that is likely not well + categorized into a small set of acceptable values. +* ``_date`` indicates the field contains a :class:`Date` object. +* ``_datetime`` indicates the field contains a full :class:`Datetime` object. +* ``_year`` indicates the field contains an :class:`integer` 4-digit year. +* ``capacity`` refers to nameplate capacity (e.g. ``capacity_mw``)-- other + specific types of capacity are annotated. +* Regardless of what label utilities are given in the original data source + (e.g. ``operator`` in EIA or ``respondent`` in FERC) we refer to them as + ``utilities`` in PUDL. + +Naming Conventions in Code +-------------------------- In the PUDL codebase, we aspire to follow the naming and other conventions detailed in :pep:`8`. @@ -13,18 +201,13 @@ as we come across them again in maintaining the code. (e.g. connect_db), unless the function returns a simple value (e.g. datadir). * No duplication of information (e.g. form names). * lowercase, underscores separate words (i.e. ``snake_case``). -* Semi-private helper functions (functions used within a single module only - and not exposed via the public API) should be preceded by an underscore. +* Add a preceeding underscore to semi-private helper functions (functions used + within a single module only and not exposed via the public API). * When the object is a table, use the full table name (e.g. ingest_fuel_ferc1). * When dataframe outputs are built from multiple tables, identify the type of information being pulled (e.g. "plants") and the source of the tables (e.g. ``eia`` or ``ferc1``). When outputs are built from a single table, simply use - the table name (e.g. ``boiler_fuel_eia923``). - -.. _glossary: - -Glossary of Abbreviations -------------------------- + the table name (e.g. ``core_eia923__monthly_boiler_fuel``). General Abbreviations ^^^^^^^^^^^^^^^^^^^^^ @@ -76,61 +259,9 @@ Abbreviation Definition Data Extraction Functions -------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^ The lower level namespace uses an imperative verb to identify the action the function performs followed by the object of extraction (e.g. ``get_eia860_file``). The upper level namespace identifies the dataset where extraction is occurring. - -Output Functions ------------------ - -When dataframe outputs are built from multiple tables, identify the type of -information being pulled (e.g. ``plants``) and the source of the tables (e.g. -``eia`` or ``ferc1``). When outputs are built from a single table, simply use -the table name (e.g. ``boiler_fuel_eia923``). - -Table Names ------------ - -See `this article `__ on database naming conventions. - -* Table names in snake_case -* The data source should follow the thing it applies to e.g. ``plant_id_ferc1`` - -Columns and Field Names ------------------------ - -* ``total`` should come at the beginning of the name (e.g. - ``total_expns_production``) -* Identifiers should be structured ``type`` + ``_id_`` + ``source`` where - ``source`` is the agency or organization that has assigned the ID. (e.g. - ``plant_id_eia``) -* The data source or label (e.g. ``plant_id_pudl``) should follow the thing it - is describing -* Units should be appended to field names where applicable (e.g. - ``net_generation_mwh``). This includes "per unit" signifiers (e.g. ``_pct`` - for percent, ``_ppm`` for parts per million, or a generic ``_per_unit`` when - the type of unit varies, as in columns containing a heterogeneous collection - of fuels) -* Financial values are assumed to be in nominal US dollars. -* ``_id`` indicates the field contains a usually numerical reference to - another table, which will not be intelligible without looking up the value in - that other table. -* The suffix ``_code`` indicates the field contains a short abbreviation from - a well defined list of values, that probably needs to be looked up if you - want to understand what it means. -* The suffix ``_type`` (e.g. ``fuel_type``) indicates a human readable category - from a well defined list of values. Whenever possible we try to use these - longer descriptive names rather than codes. -* ``_name`` indicates a longer human readable name, that is likely not well - categorized into a small set of acceptable values. -* ``_date`` indicates the field contains a :class:`Date` object. -* ``_datetime`` indicates the field contains a full :class:`Datetime` object. -* ``_year`` indicates the field contains an :class:`integer` 4-digit year. -* ``capacity`` refers to nameplate capacity (e.g. ``capacity_mw``)-- other - specific types of capacity are annotated. -* Regardless of what label utilities are given in the original data source - (e.g. ``operator`` in EIA or ``respondent`` in FERC) we refer to them as - ``utilities`` in PUDL. diff --git a/docs/dev/run_the_etl.rst b/docs/dev/run_the_etl.rst index 19139f56a1..b6cd2b0058 100644 --- a/docs/dev/run_the_etl.rst +++ b/docs/dev/run_the_etl.rst @@ -469,7 +469,7 @@ The commands above should result in a bunch of Python :mod:`logging` output desc what the script is doing, and file outputs the directory you specified via the ``$PUDL_OUTPUT`` environment variable. When the ETL is complete, you should see new files at e.g. ``$PUDL_OUTPUT/ferc1_dbf.sqlite``, ``$PUDL_OUTPUT/pudl.sqlite`` and -``$PUDL_OUTPUT/hourly_emissions_epacems.parquet``. +``$PUDL_OUTPUT/core_epacems__hourly_emissions.parquet``. If you need to re-run ``ferc_to_sqlite`` and want to overwrite their previous outputs you can add ``--clobber`` (run ``ferc_to_sqlite --clobber``). All of the PUDL scripts diff --git a/docs/index.rst b/docs/index.rst index f7904d4a97..fc2ab28f78 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,14 +2,221 @@ The Public Utility Data Liberation Project =============================================================================== -.. include:: ../README.rst - :start-after: readme-intro +PUDL is a data processing pipeline created by `Catalyst Cooperative +`__ that cleans, integrates, and standardizes some of the most +widely used public energy datasets in the US. The data serve researchers, activists, +journalists, and policy makers that might not have the technical expertise to access it +in its raw form, the time to clean and prepare the data for bulk analysis, or the means +to purchase it from existing commercial providers. + +--------------------------------------------------------------------------------------- +Available Data +--------------------------------------------------------------------------------------- + +We focus primarily on poorly curated data published by the US government in +semi-structured but machine readable formats. For details on exactly what data is +available from these data sources and what state it is in, see the the individual +pages for each source: + +* :doc:`data_sources/eia860` +* :doc:`data_sources/eia861` +* :doc:`data_sources/eia923` +* :doc:`data_sources/epacems` +* :doc:`data_sources/ferc1` +* :doc:`data_sources/ferc714` + +PUDL's clean and complete versions of these data sources are stored in the +``pudl.sqlite`` database. Larger datasets like EPA CEMS are stored in parquet files. +To get started using PUDL data, visit our :doc:`data_access` page, or continue reading +to learn more about the PUDL data processing pipeline. + +We also publish SQLite databases containing relatively pristine versions of our more +difficult to parse inputs, especially the old Visual FoxPro (DBF, pre-2021) and new XBRL +data (2021+) published by FERC: + +* `FERC Form 1 (DBF) `__ +* `FERC Form 1 (XBRL) `__ +* `FERC Form 2 (DBF) `__ +* `FERC Form 2 (XBRL) `__ +* `FERC Form 6 (DBF) `__ +* `FERC Form 6 (XBRL) `__ +* `FERC Form 60 (DBF) `__ +* `FERC Form 60 (XBRL) `__ +* `FERC Form 714 (XBRL) `__ + +.. _raw-data-archive: + +--------------------------------------------------------------------------------------- +Raw Data Archives +--------------------------------------------------------------------------------------- + +PUDL depends on "raw" data inputs from sources that are known to occasionally update +their data or alter the published format. These changes may be incompatible with the way +the data are read and interpreted by PUDL, so, to ensure the integrity of our data +processing, we periodically create archives of `the raw inputs on Zenodo +`__. Each of the data inputs may +have several different versions archived, and all are assigned a unique DOI and made +available through the REST API. Each release of the PUDL Python package is embedded +with a set of of DOIs to indicate which version of the raw inputs it is meant to +process. This process helps ensure that our outputs are replicable. + +To enable programmatic access to individual partitions of the data (by year, state, +etc.), we archive the raw inputs as `Frictionless Data Packages +`__. The data packages contain both the +raw data in their originally published format (CSVs, Excel spreadsheets, and Visual +FoxPro database (DBF) files) and metadata that describes how each the +dataset is partitioned. + +The PUDL software will download a copy of the appropriate raw inputs automatically as +needed and organize them in a local :doc:`datastore `. + +.. seealso:: + + The software that creates and archives the raw inputs can be found in our + `PUDL Archiver `__ + repository on GitHub. + +.. _etl-process: + +--------------------------------------------------------------------------------------- +The ETL Process +--------------------------------------------------------------------------------------- + +PUDL's ETL produces a data warehouse that can be used for analytics. +The processing happens within Dagster assets that are persisted to storage, +typically pickle, parquet or SQLite files. The raw data moves through three +layers of processing. + +Raw Layer +^^^^^^^^^ + +Assets in the Raw layer read the raw data from the original heterogeneous formats into +a collection of :class:`pandas.DataFrame` with uniform column names across all years so +that it can be easily processed in bulk. Data distributed as binary database files, such +as the DBF files from FERC Form 1, may be converted into a unified SQLite database +before individual dataframes are created. Raw data assets are not written to +``pudl.sqlite``. Instead they are persisted to pickle files and not distributed +to users. + +.. seealso:: + + Module documentation within the :mod:`pudl.extract` subpackage. + +Core Layer +^^^^^^^^^^ + +The Core layer contains well-modeled assets that serve as building blocks for +downstream wide tables and analyses. Well-modeled means tables in the database +have logical primary keys, foreign keys, datatypes and generally follow +:ref:`Tidy Data standards `. The assets are loaded to a SQLite +database or Parquet file. + +These outputs can be accessed via Python, R, and many other tools. See the +:doc:`data_dictionaries/pudl_db` page for a list of the normalized database tables and +their contents. + +Data processing in the Core layer is generally broken down into two phases. Phase one +focuses on cleaning and organizing data within individual tables while phase two focuses +on the integration and deduplication of data between tables. These tasks can be tedious +`data wrangling toil `__ that impose a +huge amount of overhead on anyone trying to do analysis based on the publicly +available data. PUDL implements common data cleaning operations in the hopes that we +can all work on more interesting problems most of the time. These operations include: + +* Standardization of units (e.g. dollars not thousands of dollars) +* Standardization of N/A values +* Standardization of freeform names and IDs +* Use of controlled vocabularies for categorical values like fuel type +* Use of more readable codes and column names +* Imposition of well defined, rich data types for each column +* Converting local timestamps to UTC +* Reshaping of data into well normalized tables which minimize data duplication +* Inferring Plant IDs which link records across many years of FERC Form 1 data +* Inferring linkages between FERC and EIA Plants and Utilities. +* Inferring more complete associations between EIA boilers and generators + +.. seealso:: + + The module and per-table transform functions in the :mod:`pudl.transform` + sub-package have more details on the specific transformations applied to each + table. + +Many of the original datasets contain large amounts of duplicated data. For instance, +the EIA reports the name of each power plant in every table that refers to otherwise +unique plant-related data. Similarly, many attributes like plant latitude and +longitude are reported separately every year. Often, these reported values are not +self-consistent. There may be several different spellings of a plant's name, or an +incorrectly reported latitude in one year. + +Assets in the Core layer attempt to eliminate this kind of inconsistent and duplicate +information when normalizing the tables by choosing only the most consistently reported +value for inclusion in the final database. If a value which should be static is not +consistently reported, it may also be set to N/A. + +Output Layer +^^^^^^^^^^^^^^^^^^^^ + +Assets in the Core layer normalize the data to make storage more efficient and avoid +data integrity issues, but you may want to combine information from more than one of +the tables to make the data more readable and readily interpretable. For example, PUDL +stores the name that EIA uses to refer to a power plant in the +:ref:`core_eia__entity_plants` table in association with the plant's unique numeric ID. +If you are working with data from the :ref:`core_eia923__monthly_fuel_receipts_costs` +table, which records monthly per-plant fuel deliveries, you may want to have the name +of the plant alongside the fuel delivery information since it's more recognizable than +the plant ID. + +Rather than requiring everyone to write their own SQL ``SELECT`` and ``JOIN`` statements +or do a bunch of :func:`pandas.merge` operations to bring together data, PUDL provides a +variety of output tables that contain all of the useful information in one place. In +some cases, like with EIA, the output tables are composed to closely resemble the raw +spreadsheet tables you're familiar with. + +The Output layer also contains tables produced by analytical routines for +calculating derived values like the heat rate by generation unit (:meth:`hr_by_unit +`) or the capacity factor by generator +(:meth:`capacity_factor `). We intend to +integrate more analytical outputs into the library over time. + +.. seealso:: + + * `The PUDL Examples GitHub repo `__ + to see how to access the PUDL Database directly, use the output functions, or + work with the EPA CEMS data using Dask. + * `How to Learn Dask in 2021 `__ + is a great collection of self-guided resources if you are already familiar with + Python, Pandas, and NumPy. + +.. _test-and-validate: + +--------------------------------------------------------------------------------------- +Data Validation +--------------------------------------------------------------------------------------- +We have a growing collection of data validation test cases that we run before +publishing a data release to try and avoid publishing data with known issues. Most of +these validations are described in the :mod:`pudl.validate` module. They check things +like: + +* The heat content of various fuel types are within expected bounds. +* Coal ash, moisture, mercury, sulfur etc. content are within expected bounds +* Generator heat rates and capacity factors are realistic for the type of prime mover + being reported. + +Some data validations are currently only specified within our test suite, including: + +* The expected number of records within each table +* The fact that there are no entirely N/A columns + +A variety of database integrity checks are also run either during the data processing +or when the data is loaded into SQLite. + +See our :doc:`dev/testing` documentation for more information. + .. toctree:: :hidden: :maxdepth: 2 - intro data_access data_sources/index data_dictionaries/index diff --git a/docs/intro.rst b/docs/intro.rst deleted file mode 100644 index c3c85c1ab5..0000000000 --- a/docs/intro.rst +++ /dev/null @@ -1,238 +0,0 @@ -======================================================================================= -Introduction -======================================================================================= - -PUDL is a data processing pipeline created by `Catalyst Cooperative -`__ that cleans, integrates, and standardizes some of the most -widely used public energy datasets in the US. The data serve researchers, activists, -journalists, and policy makers that might not have the technical expertise to access it -in its raw form, the time to clean and prepare the data for bulk analysis, or the means -to purchase it from existing commercial providers. - ---------------------------------------------------------------------------------------- -Available Data ---------------------------------------------------------------------------------------- - -We focus primarily on poorly curated data published by the US government in -semi-structured but machine readable formats. For details on exactly what data is -available from these data sources and what state it is in, see the the individual -pages for each source: - -* :doc:`data_sources/eia860` -* :doc:`data_sources/eia861` -* :doc:`data_sources/eia923` -* :doc:`data_sources/epacems` -* :doc:`data_sources/ferc1` -* :doc:`data_sources/ferc714` - -We also publish SQLite databases containing relatively pristine versions of our more -difficult to parse inputs, especially the old Visual FoxPro (DBF, pre-2021) and new XBRL -data (2021+) published by FERC: - -* `FERC Form 1 (DBF) `__ -* `FERC Form 1 (XBRL) `__ -* `FERC Form 2 (DBF) `__ -* `FERC Form 2 (XBRL) `__ -* `FERC Form 6 (DBF) `__ -* `FERC Form 6 (XBRL) `__ -* `FERC Form 60 (DBF) `__ -* `FERC Form 60 (XBRL) `__ -* `FERC Form 714 (XBRL) `__ - -To get started using PUDL data, visit our :doc:`data_access` page, or continue reading -to learn more about the PUDL data processing pipeline. - -.. _raw-data-archive: - ---------------------------------------------------------------------------------------- -Raw Data Archives ---------------------------------------------------------------------------------------- - -PUDL depends on "raw" data inputs from sources that are known to occasionally update -their data or alter the published format. These changes may be incompatible with the way -the data are read and interpreted by PUDL, so, to ensure the integrity of our data -processing, we periodically create archives of `the raw inputs on Zenodo -`__. Each of the data inputs may -have several different versions archived, and all are assigned a unique DOI and made -available through the REST API. Each release of the PUDL Python package is embedded -with a set of of DOIs to indicate which version of the raw inputs it is meant to -process. This process helps ensure that our outputs are replicable. - -To enable programmatic access to individual partitions of the data (by year, state, -etc.), we archive the raw inputs as `Frictionless Data Packages -`__. The data packages contain both the -raw data in their originally published format (CSVs, Excel spreadsheets, and Visual -FoxPro database (DBF) files) and metadata that describes how each the -dataset is partitioned. - -The PUDL software will download a copy of the appropriate raw inputs automatically as -needed and organize them in a local :doc:`datastore `. - -.. seealso:: - - The software that creates and archives the raw inputs can be found in our - `PUDL Archiver `__ - repository on GitHub. - -.. _etl-process: - ---------------------------------------------------------------------------------------- -The ETL Process ---------------------------------------------------------------------------------------- - -The core of PUDL's work takes place in the ETL (Extract, Transform, and Load) -process. - -Extract -^^^^^^^ - -The Extract step reads the raw data from the original heterogeneous formats into a -collection of :class:`pandas.DataFrame` with uniform column names across all years so -that it can be easily processed in bulk. Data distributed as binary database files, such -as the DBF files from FERC Form 1, may be converted into a unified SQLite database -before individual dataframes are created. - -.. seealso:: - - Module documentation within the :mod:`pudl.extract` subpackage. - -Transform -^^^^^^^^^ - -The Transform step is generally broken down into two phases. Phase one focuses on -cleaning and organizing data within individual tables while phase two focuses on the -integration and deduplication of data between tables. These tasks can be tedious -`data wrangling toil `__ that impose a -huge amount of overhead on anyone trying to do analysis based on the publicly -available data. PUDL implements common data cleaning operations in the hopes that we -can all work on more interesting problems most of the time. These operations include: - -* Standardization of units (e.g. dollars not thousands of dollars) -* Standardization of N/A values -* Standardization of freeform names and IDs -* Use of controlled vocabularies for categorical values like fuel type -* Use of more readable codes and column names -* Imposition of well defined, rich data types for each column -* Converting local timestamps to UTC -* Reshaping of data into well normalized tables which minimize data duplication -* Inferring Plant IDs which link records across many years of FERC Form 1 data -* Inferring linkages between FERC and EIA Plants and Utilities. -* Inferring more complete associations between EIA boilers and generators - -.. seealso:: - - The module and per-table transform functions in the :mod:`pudl.transform` - sub-package have more details on the specific transformations applied to each - table. - -Many of the original datasets contain large amounts of duplicated data. For instance, -the EIA reports the name of each power plant in every table that refers to otherwise -unique plant-related data. Similarly, many attributes like plant latitude and -longitude are reported separately every year. Often, these reported values are not -self-consistent. There may be several different spellings of a plant's name, or an -incorrectly reported latitude in one year. - -The transform step attempts to eliminate this kind of inconsistent and duplicate -information when normalizing the tables by choosing only the most consistently reported -value for inclusion in the final database. If a value which should be static is not -consistently reported, it may also be set to N/A. - -.. seealso:: - - * `Tidy Data `__ by Hadley - Wickham, Journal of Statistical Software (2014). - * `A Simple Guide to the Five Normal Forms in Relational Database Theory `__ - by William Kent, Communications of the ACM (1983). - -Load -^^^^ - -At the end of the Transform step, we have collections of :class:`pandas.DataFrame` that -correspond to database tables. These are loaded into a SQLite database. -To handle the ~1 billion row :doc:`data_sources/epacems`, we load the dataframes into -an Apache Parquet dataset that is partitioned by state and year. - -These outputs can be accessed via Python, R, and many other tools. See the -:doc:`data_dictionaries/pudl_db` page for a list of the normalized database tables and -their contents. - -.. seealso:: - - Module documentation within the :mod:`pudl.load` sub-package. - -.. _db-and-outputs: - ---------------------------------------------------------------------------------------- -Output Tables ---------------------------------------------------------------------------------------- - -Denormalized Outputs -^^^^^^^^^^^^^^^^^^^^ - -We normalize the data to make storage more efficient and avoid data integrity issues, -but you may want to combine information from more than one of the tables to make the -data more readable and readily interpretable. For example, PUDL stores the name that EIA -uses to refer to a power plant in the :ref:`plants_entity_eia` table in association with -the plant's unique numeric ID. If you are working with data from the -:ref:`fuel_receipts_costs_eia923` table, which records monthly per-plant fuel -deliveries, you may want to have the name of the plant alongside the fuel delivery -information since it's more recognizable than the plant ID. - -Rather than requiring everyone to write their own SQL ``SELECT`` and ``JOIN`` statements -or do a bunch of :func:`pandas.merge` operations to bring together data, PUDL provides a -variety of predefined queries as methods of the :class:`pudl.output.pudltabl.PudlTabl` -class. These methods perform common joins to return output tables (pandas DataFrames) -that contain all of the useful information in one place. In some cases, like with EIA, -the output tables are composed to closely resemble the raw spreadsheet tables you're -familiar with. - -.. note:: - - In the future, we intend to replace the simple denormalized output tables with - database views that are integrated into the distributed SQLite database directly. - This will provide the same convenience without requiring use of the Python software - layer. - -Analysis Outputs -^^^^^^^^^^^^^^^^ - -There are several analytical routines built into the -:mod:`pudl.output.pudltabl.PudlTabl` output objects for calculating derived values -like the heat rate by generation unit (:meth:`hr_by_unit -`) or the capacity factor by generator -(:meth:`capacity_factor `). We intend to -integrate more analytical outputs into the library over time. - -.. seealso:: - - * `The PUDL Examples GitHub repo `__ - to see how to access the PUDL Database directly, use the output functions, or - work with the EPA CEMS data using Dask. - * `How to Learn Dask in 2021 `__ - is a great collection of self-guided resources if you are already familiar with - Python, Pandas, and NumPy. - -.. _test-and-validate: - ---------------------------------------------------------------------------------------- -Data Validation ---------------------------------------------------------------------------------------- -We have a growing collection of data validation test cases that we run before -publishing a data release to try and avoid publishing data with known issues. Most of -these validations are described in the :mod:`pudl.validate` module. They check things -like: - -* The heat content of various fuel types are within expected bounds. -* Coal ash, moisture, mercury, sulfur etc. content are within expected bounds -* Generator heat rates and capacity factors are realistic for the type of prime mover - being reported. - -Some data validations are currently only specified within our test suite, including: - -* The expected number of records within each table -* The fact that there are no entirely N/A columns - -A variety of database integrity checks are also run either during the ETL process or -when the data is loaded into SQLite. - -See our :doc:`dev/testing` documentation for more information. diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 06cd8d17a1..0162000897 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -89,6 +89,22 @@ Dagster Adoption * :mod:`pudl.convert.censusdp1tract_to_sqlite` and :mod:`pudl.output.censusdp1tract` are now integrated into dagster. See :issue:`1973` and :pr:`2621`. +New Asset Naming Convention +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +There are hundreds of new tables in ``pudl.sqlite`` now that the methods in ``PudlTabl`` +have been converted to Dagster assets. This significant increase in tables and diversity +of table types prompted us to create a new naming convention to make the table names +more descriptive and organized. You can read about the new naming convention in the +:ref:`docs `. + +To help users migrate away from using ``PudlTabl`` and our temporary table names, +we've created a `google sheet `__ +that maps the old table names and ``PudlTabl`` methods to the new table names. + +We've added deprecation warnings to the ``PudlTabl`` class. We plan to remove +``PudlTabl`` from the ``pudl`` package once our known users have +succesfully migrated to pulling data directly from ``pudl.sqlite``. + Data Coverage ^^^^^^^^^^^^^ @@ -103,51 +119,63 @@ Data Coverage CAMPD API, and to include 2022 data. Due to changes in the ETL, Alaska, Puerto Rico and Hawaii are now included in CEMS processing. See issue :issue:`1264` & PRs :pr:`2779`, :pr:` 2816`. -* New :ref:`epacamd_eia` crosswalk version v0.3, see issue :issue:`2317` and PR - :pr:`2316`. EPA's updates add manual matches and exclusions focusing on operating - units with a generator ID as of 2018. +* New :ref:`core_epa__assn_eia_epacamd` crosswalk version v0.3, see issue :issue:`2317` + and PR :pr:`2316`. EPA's updates add manual matches and exclusions focusing on + operating units with a generator ID as of 2018. * New PUDL tables from :doc:`data_sources/ferc1`, integrating older DBF and newer XBRL data. See :issue:`1574` for an overview of our progress integrating FERC's XBRL data. To see which DBF and XBRL tables the following PUDL tables are derived from, refer to :py:const:`pudl.extract.ferc1.TABLE_NAME_MAP` - * :ref:`electric_energy_sources_ferc1`, see issue :issue:`1819` & PR :pr:`2094`. - * :ref:`electric_energy_dispositions_ferc1`, see issue :issue:`1819` & PR :pr:`2100`. - * :ref:`transmission_statistics_ferc1`, see issue :issue:`1822` & PR :pr:`2103` - * :ref:`utility_plant_summary_ferc1`, see issue :issue:`1806` & PR :pr:`2105`. - * :ref:`balance_sheet_assets_ferc1`, see issue :issue:`1805` & PRs :pr:`2112,2127`. - * :ref:`balance_sheet_liabilities_ferc1`, see issue :issue:`1810` & PR :pr:`2134`. - * :ref:`depreciation_amortization_summary_ferc1`, see issue :issue:`1816` & PR - :pr:`2143`. - * :ref:`income_statement_ferc1`, see issue :issue:`1813` & PR :pr:`2147`. - * :ref:`electric_plant_depreciation_changes_ferc1` see issue :issue:`1808` & - :pr:`2119`. - * :ref:`electric_plant_depreciation_functional_ferc1` see issue :issue:`1808` & PR - :pr:`2183` - * :ref:`electric_operating_expenses_ferc1`, see issue :issue:`1817` & PR :pr:`2162`. - * :ref:`retained_earnings_ferc1`, see issue :issue:`1811` & PR :pr:`2155`. - * :ref:`cash_flow_ferc1`, see issue :issue:`1821` & PR :pr:`2184` - * :ref:`electricity_sales_by_rate_schedule_ferc1`, see issue :issue:`1823` & PR - :pr:`2205` + * :ref:`core_ferc1__yearly_energy_sources_sched401`, see issue :issue:`1819` & PR + :pr:`2094`. + * :ref:`core_ferc1__yearly_energy_dispositions_sched401`, see issue :issue:`1819` & + PR :pr:`2100`. + * :ref:`core_ferc1__yearly_transmission_lines_sched422`, see issue :issue:`1822` & PR + :pr:`2103` + * :ref:`core_ferc1__yearly_utility_plant_summary_sched200`, see issue + :issue:`1806` & PR :pr:`2105`. + * :ref:`core_ferc1__yearly_balance_sheet_assets_sched110`, see issue :issue:`1805` & + PRs :pr:`2112,2127`. + * :ref:`core_ferc1__yearly_balance_sheet_liabilities_sched110`, see issue + :issue:`1810` & PR :pr:`2134`. + * :ref:`core_ferc1__yearly_depreciation_summary_sched336`, see issue :issue:`1816` + & PR :pr:`2143`. + * :ref:`core_ferc1__yearly_income_statements_sched114`, see issue :issue:`1813` & PR + :pr:`2147`. + * :ref:`core_ferc1__yearly_depreciation_changes_sched219` see issue + :issue:`1808` & :pr:`2119`. + * :ref:`core_ferc1__yearly_depreciation_by_function_sched219` see issue + :issue:`1808` & PR :pr:`2183`. + * :ref:`core_ferc1__yearly_operating_expenses_sched320`, see issue :issue:`1817` & PR + :pr:`2162`. + * :ref:`core_ferc1__yearly_retained_earnings_sched118`, see issue :issue:`1811` & PR + :pr:`2155`. + * :ref:`core_ferc1__yearly_cash_flows_sched120`, see issue :issue:`1821` & PR + :pr:`2184`. + * :ref:`core_ferc1__yearly_sales_by_rate_schedules_sched304`, see issue + :issue:`1823` & PR :pr:`2205`. + * Harvested owner utilities from the EIA 860 ownership table which are now included in - the :ref:`utilities_entity_eia` and :ref:`utilities_eia` tables. See :pr:`2714` and - :pr:`2903`. Renamed columns with owner or operator suffix to differentiate between - owner and operator utility columns in :ref:`ownership_eia860` and - :ref:`denorm_ownership_eia860`. + the :ref:`core_eia__entity_utilities` and :ref:`core_pudl__assn_eia_pudl_utilities` + tables. See :pr:`2714`. Renamed columns with owner or operator suffix to differentiate + between owner and operator utility columns in :ref:`core_eia860__scd_ownership` and + :ref:`out_eia860__yearly_ownership`. See :pr:`2903`. * New PUDL tables from :doc:`data_sources/eia860`: - * :ref:`emissions_control_equipment_eia860`, see issue :issue:`2338` & PR :pr:`2561`. - * :ref:`denorm_emissions_control_equipment_eia860`, see issue :issue:`2338` & PR + * :ref:`core_eia860__scd_emissions_control_equipment`, see issue :issue:`2338` & PR :pr:`2561`. - * :ref:`boiler_emissions_control_equipment_assn_eia860`, see :issue:`2338` & PR + * :ref:`out_eia860__yearly_emissions_control_equipment`, see issue :issue:`2338` & PR :pr:`2561`. - * :ref:`boiler_cooling_assn_eia860`, see :issue:`2586` & PR :pr:`2587` - * :ref:`boiler_stack_flue_assn_eia860`, see :issue:`2586` & PR :pr:`2587` + * :ref:`core_eia860__assn_yearly_boiler_emissions_control_equipment`, see + :issue:`2338` & PR :pr:`2561`. + * :ref:`core_eia860__assn_boiler_cooling`, see :issue:`2586` & PR :pr:`2587` + * :ref:`core_eia860__assn_boiler_stack_flue`, see :issue:`2586` & PR :pr:`2587` -* The :ref:`boilers_eia860` table now includes annual boiler attributes from +* The :ref:`core_eia860__scd_boilers` table now includes annual boiler attributes from :doc:`data_sources/eia860` Schedule 6.2 Environmental Equipment data, and the new - :ref:`boilers_entity_eia` table now includes static boiler attributes. See issue + :ref:`core_eia__entity_boilers` table now includes static boiler attributes. See issue :issue:`1162` & PR :pr:`2319`. * All :doc:`data_sources/eia861` tables are now being loaded into the PUDL DB, rather than only being available via an ad-hoc ETL process that was only accessible through @@ -160,55 +188,56 @@ Data Coverage data types, rather than actually doing the ETL. See :issue:`2265` & :pr:`2403`. The newly accessible tables contain data from 2001-2021 and include: - * :ref:`advanced_metering_infrastructure_eia861` - * :ref:`balancing_authority_eia861` - * :ref:`balancing_authority_assn_eia861` - * :ref:`demand_response_eia861` - * :ref:`demand_response_water_heater_eia861` - * :ref:`demand_side_management_sales_eia861` - * :ref:`demand_side_management_ee_dr_eia861` - * :ref:`demand_side_management_misc_eia861` - * :ref:`distributed_generation_tech_eia861` - * :ref:`distributed_generation_fuel_eia861` - * :ref:`distributed_generation_misc_eia861` - * :ref:`distribution_systems_eia861` - * :ref:`dynamic_pricing_eia861` - * :ref:`energy_efficiency_eia861` - * :ref:`green_pricing_eia861` - * :ref:`mergers_eia861` - * :ref:`net_metering_customer_fuel_class_eia861` - * :ref:`net_metering_misc_eia861` - * :ref:`non_net_metering_customer_fuel_class_eia861` - * :ref:`non_net_metering_misc_eia861` - * :ref:`operational_data_revenue_eia861` - * :ref:`operational_data_misc_eia861` - * :ref:`reliability_eia861` - * :ref:`sales_eia861` - * :ref:`service_territory_eia861` - * :ref:`utility_assn_eia861` - * :ref:`utility_data_nerc_eia861` - * :ref:`utility_data_rto_eia861` - * :ref:`utility_data_misc_eia861` + * :ref:`core_eia861__yearly_advanced_metering_infrastructure` + * :ref:`core_eia861__yearly_balancing_authority` + * :ref:`core_eia861__assn_balancing_authority` + * :ref:`core_eia861__yearly_demand_response` + * :ref:`core_eia861__yearly_demand_response_water_heater` + * :ref:`core_eia861__yearly_demand_side_management_sales` + * :ref:`core_eia861__yearly_demand_side_management_ee_dr` + * :ref:`core_eia861__yearly_demand_side_management_misc` + * :ref:`core_eia861__yearly_distributed_generation_tech` + * :ref:`core_eia861__yearly_distributed_generation_fuel` + * :ref:`core_eia861__yearly_distributed_generation_misc` + * :ref:`core_eia861__yearly_distribution_systems` + * :ref:`core_eia861__yearly_dynamic_pricing` + * :ref:`core_eia861__yearly_energy_efficiency` + * :ref:`core_eia861__yearly_green_pricing` + * :ref:`core_eia861__yearly_mergers` + * :ref:`core_eia861__yearly_net_metering_customer_fuel_class` + * :ref:`core_eia861__yearly_net_metering_misc` + * :ref:`core_eia861__yearly_non_net_metering_customer_fuel_class` + * :ref:`core_eia861__yearly_non_net_metering_misc` + * :ref:`core_eia861__yearly_operational_data_revenue` + * :ref:`core_eia861__yearly_operational_data_misc` + * :ref:`core_eia861__yearly_reliability` + * :ref:`core_eia861__yearly_sales` + * :ref:`core_eia861__yearly_service_territory` + * :ref:`core_eia861__assn_utility` + * :ref:`core_eia861__yearly_utility_data_nerc` + * :ref:`core_eia861__yearly_utility_data_rto` + * :ref:`core_eia861__yearly_utility_data_misc` * A couple of tables from :doc:`data_sources/ferc714` have been added to the PUDL DB. These tables contain data from 2006-2020 (2021 is distributed by FERC in XBRL format and we have not yet integrated it). See :issue:`2266`, :pr:`2421` and :pr:`2550`. The newly accessible tables include: - * :ref:`respondent_id_ferc714` (linking FERC-714 respondents to EIA utilities) - * :ref:`demand_hourly_pa_ferc714` (hourly electricity demand by planning area) - * :ref:`fipsified_respondents_ferc714` (annual respondents with county FIPS IDs) - * :ref:`summarized_demand_ferc714` (annual demand for FERC-714 respondents) + * :ref:`core_ferc714__respondent_id` (linking FERC-714 respondents to EIA utilities) + * :ref:`core_ferc714__hourly_demand_pa` (hourly electricity demand by planning area) + * :ref:`out_ferc714__respondents_with_fips` (annual respondents with county FIPS IDs) + * :ref:`out_ferc714__summarized_demand` (annual demand for FERC-714 respondents) -* Added new table :ref:`epacamd_eia_subplant_ids`, which aguments the - :ref:`epacamd_eia` glue table. This table incorporates all - :ref:`generators_entity_eia` and all :ref:`hourly_emissions_epacems` ID's and uses - these complete IDs to develop a full-coverage ``subplant_id`` column which granularly - connects EPA CAMD with EIA. Thanks to :user:`grgmiller` for his contribution to this - process. See :issue:`2456` & :pr:`2491`. +* Added new table :ref:`core_epa__assn_eia_epacamd_subplant_ids`, which aguments the + :ref:`core_epa__assn_eia_epacamd` glue table. This table incorporates all + :ref:`core_eia__entity_generators` and all :ref:`core_epacems__hourly_emissions` ID's + and uses these complete IDs to develop a full-coverage ``subplant_id`` column which + granularly connects EPA CAMD with EIA. Thanks to :user:`grgmiller` for his + contribution to this process. See :issue:`2456` & :pr:`2491`. -* Added new table :ref:`out__yearly_plants_all_ferc1_plant_parts_eia` which links FERC1 - records from :ref:`denorm_plants_all_ferc1` and :ref:`plant_parts_eia`. +* Added new table :ref:`out_pudl__yearly_assn_eia_ferc1_plant_parts` which links FERC1 + records from :ref:`out_ferc1__yearly_all_plants` and + :ref:`out_eia__yearly_plant_parts`. * Thanks to contributions from :user:`rousik` we've generalized the code we use to convert FERC's old annual Visual FoxPro databases into multi-year SQLite databases. @@ -234,29 +263,36 @@ Data Cleaning * Removed inconsistently reported leading zeroes from numeric ``boiler_id`` values. This affected a small number of records in any table referring to boilers, including - :ref:`boilers_entity_eia`, :ref:`boilers_eia860`, :ref:`boiler_fuel_eia923`, - :ref:`boiler_generator_assn_eia860` and the :ref:`epacamd_eia` crosswalk. It - also had some minor downstream effects on the MCOE outputs. See :issue:`2366` and - :pr:`2367`. -* The :ref:`boiler_fuel_eia923` table now includes the ``prime_mover_code`` column. This - column was previously incorrectly being associated with boilers in the - :ref:`boilers_entity_eia` table. See issue :issue:`2349` & PR :pr:`2362`. -* Fixed column naming issues in the :ref:`electric_operating_revenues_ferc1` table. -* Made minor calculation fixes in the metadata for :ref:`income_statement_ferc1`, - :ref:`utility_plant_summary_ferc1`, :ref:`electric_operating_revenues_ferc1`, - :ref:`balance_sheet_assets_ferc1`, :ref:`balance_sheet_liabilities_ferc1`, and - :ref:`electric_operating_expenses_ferc1`, - :ref:`electric_plant_depreciation_changes_ferc1` and - :ref:`electric_plant_depreciation_functional_ferc1`. See :issue:`2016`, :pr:`2563`, - :pr:`2662` and :pr:`2687`. -* Changed the :ref:`retained_earnings_ferc1` table transform to restore factoids for - previous year balances, and added calculation metadata. See :issue:`1811`, - :issue:`2016`, and :pr:`2645`. + :ref:`core_eia__entity_boilers`, :ref:`core_eia860__scd_boilers`, + :ref:`core_eia923__monthly_boiler_fuel`, :ref:`core_eia860__assn_boiler_generator` + and the :ref:`core_epa__assn_eia_epacamd` crosswalk. It also had some minor downstream + effects on the MCOE outputs. See :issue:`2366` and :pr:`2367`. +* The :ref:`core_eia923__monthly_boiler_fuel` table now includes the + ``prime_mover_code`` column. This column was previously incorrectly being associated + with boilers in the :ref:`core_eia__entity_boilers` table. See issue :issue:`2349` & + PR :pr:`2362`. +* Fixed column naming issues in the + :ref:`core_ferc1__yearly_operating_revenues_sched300` table. +* Made minor calculation fixes in the metadata for + :ref:`core_ferc1__yearly_income_statements_sched114`, + :ref:`core_ferc1__yearly_utility_plant_summary_sched200`, + :ref:`core_ferc1__yearly_operating_revenues_sched300`, + :ref:`core_ferc1__yearly_balance_sheet_assets_sched110`, + :ref:`core_ferc1__yearly_balance_sheet_liabilities_sched110`, and + :ref:`core_ferc1__yearly_operating_expenses_sched320`, + :ref:`core_ferc1__yearly_depreciation_changes_sched219` and + :ref:`core_ferc1__yearly_depreciation_by_function_sched219`. See :issue:`2016`, + :pr:`2563`, :pr:`2662` and :pr:`2687`. +* Changed the :ref:`core_ferc1__yearly_retained_earnings_sched118` table transform to + restore factoids for previous year balances, and added calculation metadata. See + :issue:`1811`, :issue:`2016`, and :pr:`2645`. * Added "correction" records to many FERC Form 1 tables where the reported totals do not match the outcomes of calculations specified in XBRL metadata (even after cleaning up the often incorrect calculation specifications!). See :issue:`2957` and :pr:`2620`. -* Flip the sign of some erroneous negative values in the :ref:`plant_in_service_ferc1` - and :ref:`utility_plant_summary_ferc1` tables. See :issue:`2599`, and :pr:`2647`. +* Flip the sign of some erroneous negative values in the + :ref:`core_ferc1__yearly_plant_in_service_sched204` and + :ref:`core_ferc1__yearly_utility_plant_summary_sched200` tables. See + :issue:`2599`, and :pr:`2647`. Analysis ^^^^^^^^ @@ -267,20 +303,20 @@ Analysis complete estimates of generator heat rates and thus fuel costs and emissions. Thanks to :user:`grgmiller` for his contribution, which was integrated by :user:`cmgosnell`! See PRs :pr:`1096,1608` and issues :issue:`1468,1478`. -* Integrated :mod:`pudl.analysis.ferc1_eia` from our RMI collaboration repo, which uses - logistic regression to match FERC1 plants data to EIA 860 records. While far from - perfect, this baseline model utilizes the manually created training data and plant IDs - to perform record linkage on the FERC1 data and EIA plant parts list created in - :mod:`pudl.analysis.plant_parts_eia`. See issue :issue:`1064` & PR :pr:`2224`. To - account for 1:m matches in the manual data, we added ``plant_match_ferc1`` as a plant - part in :mod:`pudl.analysis.plant_parts_eia`. +* Integrated :mod:`pudl.analysis.eia_ferc1_record_linkage` from our RMI collaboration + repo, which uses logistic regression to match FERC1 plants data to EIA 860 records. + While far from perfect, this baseline model utilizes the manually created training + data and plant IDs to perform record linkage on the FERC1 data and EIA plant parts + list created in :mod:`pudl.analysis.plant_parts_eia`. See issue :issue:`1064` & PR + :pr:`2224`. To account for 1:m matches in the manual data, we added + ``plant_match_ferc1`` as a plant part in :mod:`pudl.analysis.plant_parts_eia`. * Refined how we are associating generation and fuel data in :mod:`pudl.analysis.allocate_gen_fuel`, which was renamed from ``allocate_net_gen``. - Energy source codes that show up in the :ref:`generation_fuel_eia923` or the - :ref:`boiler_fuel_eia923` are now added into the :ref:`generators_eia860` table so - associating those gf and bf records are more cleanly associated with generators. - Thanks to :user:`grgmiller` for his contribution, which was integrated by - :user:`cmgosnell`! See PRs :pr:`2235,2446`. + Energy source codes that show up in the :ref:`core_eia923__monthly_generation_fuel` or + the :ref:`core_eia923__monthly_boiler_fuel` are now added into the + :ref:`core_eia860__scd_generators` table so associating those gf and bf records are + more cleanly associated with generators. Thanks to :user:`grgmiller` for his + contribution, which was integrated by :user:`cmgosnell`! See PRs :pr:`2235,2446`. * The :mod:`pudl.analysis.mcoe` table now uses the allocated estimates for per-generator net generation and fuel consumption. See PR :pr:`2553`. * Additionally, the :mod:`pudl.analysis.mcoe` table now only includes attributes @@ -291,9 +327,10 @@ Analysis * Added outputs from :mod:`pudl.analysis.service_territory` and :mod:`pudl.analysis.state_demand` into PUDL. These outputs include the US Census geometries associated with balancing authority and utility data from EIA 861 - (:ref:`compiled_geometry_balancing_authority_eia861` and - :ref:`compiled_geometry_utility_eia861`), and the estimated total hourly electricity - demand for each US state in :ref:`predicted_state_hourly_demand`. See :issue:`1973` + (:ref:`out_eia861__compiled_geometry_balancing_authorities` and + :ref:`out_eia861__compiled_geometry_utilities`), and the estimated total hourly + electricity demand for each US state in + :ref:`out_ferc714__hourly_predicted_state_demand`. See :issue:`1973` and :pr:`2550`. Deprecations @@ -302,7 +339,7 @@ Deprecations * Replace references to deprecated ``pudl-scrapers`` and ``pudl-zenodo-datastore`` repositories with references to `pudl-archiver `__ repository in - :doc:`intro`, :doc:`dev/datastore`, and :doc:`dev/existing_data_updates`. See + :doc:`dev/datastore`, and :doc:`dev/existing_data_updates`. See :pr:`2190`. * :mod:`pudl.etl` is now a subpackage that collects all pudl assets into a dagster `Definition `__. All @@ -384,9 +421,9 @@ Data Coverage * Incorporated 2022 data from the :doc:`data_sources/eia860` monthly update from September 2022. See :pr:`2079`. A June 2022 eia860m update included adding new ``energy_storage_capacity_mwh`` (for batteries) and ``net_capacity_mwdc`` (for - behind-the-meter solar PV) attributes to the :ref:`generators_eia860` table, as they + behind-the-meter solar PV) attributes to the ``generators_eia860`` table, as they appear in the :doc:`data_sources/eia860` monthly updates for 2022. See :pr:`1834`. -* Added new :ref:`datasources` table, which includes partitions used to generate the +* Added new ``datasources`` table, which includes partitions used to generate the database. See :pr:`2079`. * Integrated several new columns into the EIA 860 and EIA 923 including several codes with coding tables (See :doc:`data_dictionaries/codes_and_labels`). :pr:`1836` @@ -407,14 +444,14 @@ Data Coverage updates in this release include: * Convert XBRL into raw sqlite database :pr:`1831` - * Build transformer infrastructure & Add :ref:`fuel_ferc1` table :pr:`1721` + * Build transformer infrastructure & Add ``fuel_ferc1`` table :pr:`1721` * Map utility XBRL and DBF utility IDs :pr:`1931` - * Add :ref:`plants_steam_ferc1` table :pr:`1881` - * Add :ref:`plants_hydro_ferc1` :pr:`1992` - * Add :ref:`plants_pumped_storage_ferc1` :pr:`2005` - * Add :ref:`purchased_power_ferc1` :pr:`2011` - * Add :ref:`plants_small_ferc1` table :pr:`2035` - * Add :ref:`plant_in_service_ferc1` table :pr:`2025` & :pr:`2058` + * Add ``plants_steam_ferc1`` table :pr:`1881` + * Add ``plants_hydro_ferc1`` :pr:`1992` + * Add ``plants_pumped_storage_ferc1`` :pr:`2005` + * Add ``purchased_power_ferc1`` :pr:`2011` + * Add ``plants_small_ferc1`` table :pr:`2035` + * Add ``plant_in_service_ferc1`` table :pr:`2025` & :pr:`2058` * Added all of the SQLite databases which we build from FERC's raw XBRL filings to our Datasette deployment. See :pr:`2095` & :issue:`2080`. Browse the published data here: @@ -458,14 +495,14 @@ Database Schema Changes time, we recategorized the ``prime_mover_code`` column in our entity resolution process to enable the rare but real variability over time. We moved the ``prime_mover_code`` column from the statically harvested/normalized data - column to an annually harvested data column (i.e. from :ref:`generators_entity_eia` - to :ref:`generators_eia860`) :pr:`1600`. See :issue:`1585` for more details. -* Created :ref:`operational_status_eia` into our static metadata tables (See + column to an annually harvested data column (i.e. from ``generators_entity_eia`` + to ``generators_eia860``) :pr:`1600`. See :issue:`1585` for more details. +* Created ``operational_status_eia`` into our static metadata tables (See :doc:`data_dictionaries/codes_and_labels`). Used these standard codes and code - fixes to clean ``operational_status_code`` in the :ref:`generators_entity_eia` + fixes to clean ``operational_status_code`` in the ``generators_entity_eia`` table. :pr:`1624` -* Moved a number of slowly changing plant attributes from the :ref:`plants_entity_eia` - table to the annual :ref:`plants_eia860` table. See :issue:`1748` and :pr:`1749`. +* Moved a number of slowly changing plant attributes from the ``plants_entity_eia`` + table to the annual ``plants_eia860`` table. See :issue:`1748` and :pr:`1749`. This was initially inspired by the desire to more accurately reproduce the aggregated fuel prices which are available in the EIA's API. Along with state, census region, month, year, and fuel type, those prices are broken down by industrial sector. @@ -485,9 +522,9 @@ Database Schema Changes * ``iso_rto_code`` * ``primary_purpose_id_naics`` -* Renamed ``grid_voltage_kv`` to ``grid_voltage_1_kv`` in the :ref:`plants_eia860` +* Renamed ``grid_voltage_kv`` to ``grid_voltage_1_kv`` in the ``plants_eia860`` table, to follow the pattern of many other multiply reported values. -* Added a :ref:`balancing_authorities_eia` coding table mapping BA codes found in the +* Added a ``balancing_authorities_eia`` coding table mapping BA codes found in the :doc:`data_sources/eia860` and :doc:`data_sources/eia923` to their names, cleaning up non-standard codes, and fixing some reporting errors for ``PACW`` vs. ``PACE`` (PacifiCorp West vs. East) based on the state associated with the plant reporting the @@ -499,9 +536,9 @@ Database Schema Changes ``emissions_unit_id_epa`` and not otherwise useful or transferable to other datasets. ``facility_id`` was removed because it is specific to EPA's internal database and does not aid in connection with other data. :pr:`1692` -* Added a new table :ref:`political_subdivisions` which consolidated various bits of +* Added a new table ``political_subdivisions`` which consolidated various bits of information about states, territories, provinces etc. that had previously been - scattered across constants stored in the codebase. The :ref:`ownership_eia860` table + scattered across constants stored in the codebase. The ``ownership_eia860`` table had a mix of state and country information stored in the same column, and to retain all of it we added a new ``owner_country_code`` column. :pr:`1966` @@ -685,7 +722,7 @@ Data Cleaning contexts. We use the same method to clean them all up now, and enforce a uniform field width with leading zero padding. This also allows us to enforce a regex pattern constraint on these fields in the database outputs. :pr:`1405,1476` -* We're now able to fill in missing values in the very useful :ref:`generators_eia860` +* We're now able to fill in missing values in the very useful ``generators_eia860`` ``technology_description`` field. Currently this is optionally available in the output layer, but we want to put more of this kind of data repair into the core database gong forward. :pr:`1075` @@ -710,8 +747,8 @@ Miscellaneous Bug Fixes ^^^^^^^^^ -* In addressing :issue:`851,1296,1325` the :ref:`generation_fuel_eia923` table was split - to create a :ref:`generation_fuel_nuclear_eia923` table since they have different +* In addressing :issue:`851,1296,1325` the ``generation_fuel_eia923`` table was split + to create a ``generation_fuel_nuclear_eia923`` table since they have different primary keys. This meant that the :meth:`pudl.output.pudltabl.PudlTabl.gf_eia923` method no longer included nuclear generation. This impacted the net generation allocation process and MCOE calculations downstream, which were expecting to have all @@ -800,33 +837,33 @@ the database normalization and entity resolution process also benefit from using natural primary keys when possible. As a result we've made some changes to the PUDL database schema, which will probably affect some users. -* We have split out a new :ref:`generation_fuel_nuclear_eia923` table from the existing - :ref:`generation_fuel_eia923` table, as nuclear generation and fuel consumption are +* We have split out a new ``generation_fuel_nuclear_eia923`` table from the existing + ``generation_fuel_eia923`` table, as nuclear generation and fuel consumption are reported at the generation unit level, rather than the plant level, requiring a different natural primary key. See :issue:`851,1296,1325`. -* Implementing a natural primary key for the :ref:`boiler_fuel_eia923` table required +* Implementing a natural primary key for the ``boiler_fuel_eia923`` table required the aggregation of a small number of records that didn't have well-defined ``prime_mover_code`` values. See :issue:`852,1306,1311`. * We repaired, aggregated, or dropped a small number of records in the - :ref:`generation_eia923` (See :issue:`1208,1248`) and - :ref:`ownership_eia860` (See :issue:`1207,1258`) tables due to null values in their + ``generation_eia923`` (See :issue:`1208,1248`) and + ``ownership_eia860`` (See :issue:`1207,1258`) tables due to null values in their primary key columns. * Many new foreign key constraints are being enforced between the EIA data tables, entity tables, and coding tables. See :issue:`1196`. * Fuel types and energy sources reported to EIA are now defined in / constrained by - the static :ref:`energy_sources_eia` table. + the static ``energy_sources_eia`` table. * The columns that indicate the mode of transport for various fuels now contain short codes rather than longer labels, and are defined in / constrained by the static - :ref:`fuel_transportation_modes_eia` table. + ``fuel_transportation_modes_eia`` table. * In the simplified FERC 1 fuel type categories, we're now using ``other`` instead of ``unknown``. * Several columns have been renamed to harmonize meanings between different tables and datasets, including: - * In :ref:`generation_fuel_eia923` and :ref:`boiler_fuel_eia923` the ``fuel_type`` and + * In ``generation_fuel_eia923`` and ``boiler_fuel_eia923`` the ``fuel_type`` and ``fuel_type_code`` columns have been replaced with ``energy_source_code``, which - appears in various forms in :ref:`generators_eia860` and - :ref:`fuel_receipts_costs_eia923`. + appears in various forms in ``generators_eia860`` and + ``fuel_receipts_costs_eia923``. * ``fuel_qty_burned`` is now ``fuel_consumed_units`` * ``fuel_qty_units`` is now ``fuel_received_units`` * ``heat_content_mmbtu_per_unit`` is now ``fuel_mmbtu_per_unit`` @@ -939,19 +976,19 @@ fuel consumption for all generators. The process has known issues and is being actively developed. See :pr:`989` Net electricity generation and fuel consumption are reported in multiple ways in -the EIA 923. The :ref:`generation_fuel_eia923` table reports both generation and +the EIA 923. The ``generation_fuel_eia923`` table reports both generation and fuel consumption, and breaks them down by plant, prime mover, and fuel. In -parallel, the :ref:`generation_eia923` table reports generation by generator, -and the :ref:`boiler_fuel_eia923` table reports fuel consumption by boiler. +parallel, the ``generation_eia923`` table reports generation by generator, +and the ``boiler_fuel_eia923`` table reports fuel consumption by boiler. -The :ref:`generation_fuel_eia923` table is more complete, but the -:ref:`generation_eia923` + :ref:`boiler_fuel_eia923` tables are more granular. -The :ref:`generation_eia923` table includes only ~55% of the total MWhs reported -in the :ref:`generation_fuel_eia923` table. +The ``generation_fuel_eia923`` table is more complete, but the +``generation_eia923`` + ``boiler_fuel_eia923`` tables are more granular. +The ``generation_eia923`` table includes only ~55% of the total MWhs reported +in the ``generation_fuel_eia923`` table. The :mod:`pudl.analysis.allocate_gen_fuel` module estimates the net electricity generation and fuel consumption attributable to individual generators based on -the more expansive reporting of the data in the :ref:`generation_fuel_eia923` +the more expansive reporting of the data in the ``generation_fuel_eia923`` table. Data Management and Archiving diff --git a/docs/templates/eia860_child.rst.jinja b/docs/templates/eia860_child.rst.jinja index 76f4d90371..80c53bb492 100644 --- a/docs/templates/eia860_child.rst.jinja +++ b/docs/templates/eia860_child.rst.jinja @@ -83,6 +83,6 @@ reading the Form Instructions from different years, linked above. Unfortunately, the reported associations are incomplete. We've been able to infer many more associations than those reported directly by EIA, and currently cover more than 95% -of the generation reported in the :ref:`generation_eia923` table. The associations -themselves are found in the :ref:`boiler_generator_assn_eia860` table. +of the generation reported in the :ref:`core_eia923__monthly_generation` table. The associations +themselves are found in the :ref:`core_eia860__assn_boiler_generator` table. {% endblock %} diff --git a/docs/templates/eia923_child.rst.jinja b/docs/templates/eia923_child.rst.jinja index 7f04f7b12f..731b52d0bc 100644 --- a/docs/templates/eia923_child.rst.jinja +++ b/docs/templates/eia923_child.rst.jinja @@ -80,10 +80,10 @@ and the commodity cost information for all plants (Schedule 2). **Net generation & fuel consumed reported in two separate tables** Net generation and fuel consumption are reported in two separate tables in EIA-923: in -the :ref:`generation_eia923` and :ref:`generation_fuel_eia923` tables. The -:ref:`generation_fuel_eia923` table is more complete (the :ref:`generation_eia923` table -includes only ~55% of the reported MWh), but the :ref:`generation_eia923` table is more -granular (it is reported at the generator level). +the :ref:`core_eia923__monthly_generation` and :ref:`core_eia923__monthly_generation_fuel` tables. The +:ref:`core_eia923__monthly_generation_fuel` table is more complete (the +:ref:`core_eia923__monthly_generation` table includes only ~55% of the reported MWh), but the +:ref:`core_eia923__monthly_generation` table is more granular (it is reported at the generator level). **Data Estimates** Plants that did not respond or reported unverified data were recorded as estimates diff --git a/docs/templates/ferc714_child.rst.jinja b/docs/templates/ferc714_child.rst.jinja index a4ab1e8aba..961b297452 100644 --- a/docs/templates/ferc714_child.rst.jinja +++ b/docs/templates/ferc714_child.rst.jinja @@ -23,13 +23,13 @@ respondents that show up in the respondents table. WIth the EIA IDs, we link the hourly electricity demand to a particular georgraphic region at the county level, because utilities and balancing authorities report their -service territories in :ref:`service_territory_eia861`, and from that information we +service territories in :ref:`core_eia861__yearly_service_territory`, and from that information we can estimate historical hourly electricity demand by state. -Plant operators reported in :ref:`plants_eia860` and generator ownership information -reported in :ref:`ownership_eia860` are linked to :ref:`utilities_eia860` and -:ref:`balancing_authority_eia861` and so can also be linked to the -:ref:`respondent_id_ferc714` table, as well as the :ref:`hourly_emissions_epacems` +Plant operators reported in :ref:`core_eia860__scd_plants` and generator ownership information +reported in :ref:`core_eia860__scd_ownership` are linked to :ref:`core_eia860__scd_utilities` and +:ref:`core_eia861__yearly_balancing_authority` and so can also be linked to the +:ref:`core_ferc714__respondent_id` table, as well as the :ref:`core_epacems__hourly_emissions` unit-level emissions and generation data reported in :doc:`epacems`. {% endblock %} diff --git a/docs/templates/package.rst.jinja b/docs/templates/package.rst.jinja index 7a450b53cb..e18365d7c9 100644 --- a/docs/templates/package.rst.jinja +++ b/docs/templates/package.rst.jinja @@ -3,11 +3,9 @@ PUDL Data Dictionary =============================================================================== The following data tables have been cleaned and transformed by our ETL process. -Please note that tables beginning with "denorm" are temporary tables whose names and -metadata will shortly change, as we migrate new tables into our database. -{% for resource in package.resources %} -.. _{{ resource.name }}: +{% for resource in package.get_sorted_resources() %} +.. _{{ resource.name|format_resource_name_cross_ref }}: {% include 'resource.rst.jinja' %} {% endfor %} diff --git a/migrations/versions/0678b90669bb_fix_ownership_harvesting.py b/migrations/versions/0678b90669bb_fix_ownership_harvesting.py deleted file mode 100644 index be0c2e1cd1..0000000000 --- a/migrations/versions/0678b90669bb_fix_ownership_harvesting.py +++ /dev/null @@ -1,32 +0,0 @@ -"""fix ownership harvesting - -Revision ID: 0678b90669bb -Revises: 0cca77e90510 -Create Date: 2023-10-02 20:18:21.141000 - -""" -from alembic import op - -# revision identifiers, used by Alembic. -revision = '0678b90669bb' -down_revision = '0cca77e90510' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_ownership_eia860_plant_id_eia_generators_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_ownership_eia860_plant_id_eia_generators_entity_eia'), 'generators_entity_eia', ['plant_id_eia', 'generator_id'], ['plant_id_eia', 'generator_id']) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_ownership_eia860_plant_id_eia_generators_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_ownership_eia860_plant_id_eia_generators_eia860', 'generators_eia860', ['plant_id_eia', 'generator_id', 'report_date'], ['plant_id_eia', 'generator_id', 'report_date']) - - # ### end Alembic commands ### diff --git a/migrations/versions/0cca77e90510_update_ownership_table_columns.py b/migrations/versions/0cca77e90510_update_ownership_table_columns.py deleted file mode 100644 index 275d1b7d80..0000000000 --- a/migrations/versions/0cca77e90510_update_ownership_table_columns.py +++ /dev/null @@ -1,33 +0,0 @@ -"""update ownership table columns - -Revision ID: 0cca77e90510 -Revises: 517265fee966 -Create Date: 2023-10-02 00:12:56.142226 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = '0cca77e90510' -down_revision = '517265fee966' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.add_column(sa.Column('owner_state', sa.Enum('AK', 'CO', 'VT', 'OK', 'NV', 'KS', 'FL', 'DC', 'GU', 'HI', 'MD', 'NS', 'MO', 'BC', 'ND', 'GA', 'NL', 'OR', 'NE', 'NJ', 'IL', 'NY', 'WA', 'WV', 'WY', 'WI', 'NT', 'MB', 'MP', 'AS', 'KY', 'VA', 'NH', 'NU', 'OH', 'IN', 'IA', 'LA', 'NC', 'DE', 'ME', 'UT', 'SC', 'ON', 'NB', 'CA', 'PE', 'CT', 'TX', 'SD', 'SK', 'AR', 'PR', 'MA', 'ID', 'QC', 'MT', 'YT', 'AL', 'RI', 'AB', 'MN', 'MI', 'NM', 'PA', 'VI', 'AZ', 'MS', 'TN'), nullable=True, comment='Two letter ISO-3166 political subdivision code.')) - batch_op.drop_column('state') - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.add_column(sa.Column('state', sa.TEXT(), nullable=True)) - batch_op.drop_column('owner_state') - - # ### end Alembic commands ### diff --git a/migrations/versions/1ceb9897fd34_add_data_maturity_to_eia923m_tables.py b/migrations/versions/1ceb9897fd34_add_data_maturity_to_eia923m_tables.py deleted file mode 100644 index ec582b8c97..0000000000 --- a/migrations/versions/1ceb9897fd34_add_data_maturity_to_eia923m_tables.py +++ /dev/null @@ -1,113 +0,0 @@ -"""add data_maturity to eia923m tables - -Revision ID: 1ceb9897fd34 -Revises: f11241c9292d -Create Date: 2023-10-26 16:30:33.771381 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = '1ceb9897fd34' -down_revision = 'f11241c9292d' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('boiler_fuel_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_boiler_fuel_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_boiler_fuel_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_boiler_fuel_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_boiler_fuel_monthly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_boiler_fuel_monthly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_boiler_fuel_yearly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_boiler_fuel_yearly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_fuel_receipts_costs_monthly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_fuel_receipts_costs_monthly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_fuel_receipts_costs_yearly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_generation_fuel_combined_monthly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_fuel_combined_monthly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_generation_fuel_combined_yearly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_fuel_combined_yearly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_generation_monthly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_monthly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_generation_yearly_eia923', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_yearly_eia923_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - with op.batch_alter_table('denorm_plants_utilities_eia', schema=None) as batch_op: - batch_op.add_column(sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.')) - batch_op.create_foreign_key(batch_op.f('fk_denorm_plants_utilities_eia_data_maturity_data_maturities'), 'data_maturities', ['data_maturity'], ['code']) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('denorm_plants_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_plants_utilities_eia_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_generation_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_yearly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_generation_monthly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_monthly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_generation_fuel_combined_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_fuel_combined_yearly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_generation_fuel_combined_monthly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_fuel_combined_monthly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_fuel_receipts_costs_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_fuel_receipts_costs_monthly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_fuel_receipts_costs_monthly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_boiler_fuel_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_boiler_fuel_yearly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_boiler_fuel_monthly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_boiler_fuel_monthly_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('denorm_boiler_fuel_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_boiler_fuel_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - with op.batch_alter_table('boiler_fuel_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_boiler_fuel_eia923_data_maturity_data_maturities'), type_='foreignkey') - batch_op.drop_column('data_maturity') - - # ### end Alembic commands ### diff --git a/migrations/versions/3313ca078f4e_demand_hourly_pa_ferc714_report_date_.py b/migrations/versions/3313ca078f4e_demand_hourly_pa_ferc714_report_date_.py deleted file mode 100644 index 61acfd18cf..0000000000 --- a/migrations/versions/3313ca078f4e_demand_hourly_pa_ferc714_report_date_.py +++ /dev/null @@ -1,35 +0,0 @@ -"""demand_hourly_pa_ferc714.report_date can't be null - -Revision ID: 3313ca078f4e -Revises: 1ceb9897fd34 -Create Date: 2023-11-02 15:48:50.477585 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = '3313ca078f4e' -down_revision = '1ceb9897fd34' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('demand_hourly_pa_ferc714', schema=None) as batch_op: - batch_op.alter_column('report_date', - existing_type=sa.DATE(), - nullable=False) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('demand_hourly_pa_ferc714', schema=None) as batch_op: - batch_op.alter_column('report_date', - existing_type=sa.DATE(), - nullable=True) - - # ### end Alembic commands ### diff --git a/migrations/versions/517265fee966_fix_owner_utility_harvesting.py b/migrations/versions/4b08158ae952_apply_naming_convention_to_all_assets.py similarity index 67% rename from migrations/versions/517265fee966_fix_owner_utility_harvesting.py rename to migrations/versions/4b08158ae952_apply_naming_convention_to_all_assets.py index 4a939a4fff..e623f2a241 100644 --- a/migrations/versions/517265fee966_fix_owner_utility_harvesting.py +++ b/migrations/versions/4b08158ae952_apply_naming_convention_to_all_assets.py @@ -1,8 +1,8 @@ -"""fix owner utility harvesting +"""Apply naming convention to all assets -Revision ID: 517265fee966 +Revision ID: 4b08158ae952 Revises: -Create Date: 2023-10-01 21:55:00.263467 +Create Date: 2023-12-13 11:50:58.385316 """ import sqlalchemy as sa @@ -10,7 +10,7 @@ from sqlalchemy.dialects import sqlite # revision identifiers, used by Alembic. -revision = '517265fee966' +revision = '4b08158ae952' down_revision = None branch_labels = None depends_on = None @@ -18,105 +18,75 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table('averaging_periods_eia', - sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), - sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), - sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_averaging_periods_eia')) - ) - op.create_table('balancing_authorities_eia', - sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), - sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), - sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_balancing_authorities_eia')) - ) - op.create_table('balancing_authority_assn_eia861', + op.create_table('core_eia861__assn_balancing_authority', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=False, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('state', sa.Text(), nullable=False, comment='Two letter US state abbreviation.'), - sa.PrimaryKeyConstraint('report_date', 'balancing_authority_id_eia', 'utility_id_eia', 'state', name=op.f('pk_balancing_authority_assn_eia861')) + sa.PrimaryKeyConstraint('report_date', 'balancing_authority_id_eia', 'utility_id_eia', 'state', name=op.f('pk_core_eia861__assn_balancing_authority')) ) - op.create_table('balancing_authority_eia861', + op.create_table('core_eia861__assn_utility', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), + sa.Column('state', sa.Text(), nullable=False, comment='Two letter US state abbreviation.'), + sa.PrimaryKeyConstraint('report_date', 'utility_id_eia', 'state', name=op.f('pk_core_eia861__assn_utility')) + ) + op.create_table('core_eia861__yearly_balancing_authority', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=False, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), sa.Column('balancing_authority_name_eia', sa.Text(), nullable=True, comment='Name of the balancing authority.'), - sa.PrimaryKeyConstraint('report_date', 'balancing_authority_id_eia', name=op.f('pk_balancing_authority_eia861')) + sa.PrimaryKeyConstraint('report_date', 'balancing_authority_id_eia', name=op.f('pk_core_eia861__yearly_balancing_authority')) ) - op.create_table('boiler_generator_assn_types_eia', + op.create_table('core_eia__codes_averaging_periods', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_boiler_generator_assn_types_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_averaging_periods')) ) - op.create_table('boiler_status_eia', + op.create_table('core_eia__codes_balancing_authorities', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_boiler_status_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_balancing_authorities')) ) - op.create_table('boiler_types_eia', + op.create_table('core_eia__codes_boiler_generator_assn_types', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_boiler_types_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_boiler_generator_assn_types')) ) - op.create_table('coalmine_types_eia', + op.create_table('core_eia__codes_boiler_status', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_coalmine_types_eia')) - ) - op.create_table('compiled_geometry_balancing_authority_eia861', - sa.Column('county_id_fips', sa.Text(), nullable=False, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), - sa.Column('county_name_census', sa.Text(), nullable=True, comment='County name as specified in Census DP1 Data.'), - sa.Column('population', sa.Float(), nullable=True, comment='County population, sourced from Census DP1 data.'), - sa.Column('area_km2', sa.Float(), nullable=True, comment='County area in km2.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=False, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('county', sa.Text(), nullable=False, comment='County name.'), - sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), - sa.PrimaryKeyConstraint('balancing_authority_id_eia', 'report_date', 'county_id_fips', 'county', name=op.f('pk_compiled_geometry_balancing_authority_eia861')) - ) - op.create_table('compiled_geometry_utility_eia861', - sa.Column('county_id_fips', sa.Text(), nullable=False, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), - sa.Column('county_name_census', sa.Text(), nullable=True, comment='County name as specified in Census DP1 Data.'), - sa.Column('population', sa.Float(), nullable=True, comment='County population, sourced from Census DP1 data.'), - sa.Column('area_km2', sa.Float(), nullable=True, comment='County area in km2.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('county', sa.Text(), nullable=True, comment='County name.'), - sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), - sa.PrimaryKeyConstraint('utility_id_eia', 'report_date', 'county_id_fips', name=op.f('pk_compiled_geometry_utility_eia861')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_boiler_status')) ) - op.create_table('contract_types_eia', + op.create_table('core_eia__codes_boiler_types', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_contract_types_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_boiler_types')) ) - op.create_table('data_maturities', + op.create_table('core_eia__codes_coalmine_types', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), + sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_data_maturities')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_coalmine_types')) ) - op.create_table('datasources', - sa.Column('datasource', sa.Enum('censusdp1tract', 'eia176', 'eia860', 'eia860m', 'eia861', 'eia923', 'eia_bulk_elec', 'eiawater', 'epacems', 'epacamd_eia', 'ferc1', 'ferc2', 'ferc6', 'ferc60', 'ferc714', 'ferceqr', 'mshamines', 'phmsagas', 'pudl'), nullable=False, comment='Code identifying a dataset available within PUDL.'), - sa.Column('partitions', sa.Text(), nullable=True, comment='The data parititions used to generate this instance of the database.'), - sa.Column('doi', sa.Text(), nullable=True, comment='Unique digitial object identifier of Zenodo archive.'), - sa.Column('pudl_version', sa.Text(), nullable=True, comment='The version of PUDL used to generate this database.'), - sa.PrimaryKeyConstraint('datasource', name=op.f('pk_datasources')) + op.create_table('core_eia__codes_contract_types', + sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), + sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), + sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_contract_types')) ) - op.create_table('emission_control_equipment_types_eia', + op.create_table('core_eia__codes_emission_control_equipment_types', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_emission_control_equipment_types_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_emission_control_equipment_types')) ) - op.create_table('energy_sources_eia', + op.create_table('core_eia__codes_energy_sources', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('fuel_units', sa.Enum('barrels', 'mcf', 'mwh', 'short_tons'), nullable=True, comment='Reported unit of measure for fuel.'), @@ -127,219 +97,310 @@ def upgrade() -> None: sa.Column('fuel_phase', sa.Enum('gas', 'liquid', 'solid'), nullable=True, comment='Physical phase of matter of the fuel.'), sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_energy_sources_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_energy_sources')) ) - op.create_table('environmental_equipment_manufacturers_eia', + op.create_table('core_eia__codes_environmental_equipment_manufacturers', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_environmental_equipment_manufacturers_eia')) - ) - op.create_table('epacamd_eia_subplant_ids', - sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_epa', sa.Integer(), nullable=True, comment='The ORISPL ID used by EPA to refer to the plant. Usually but not always the same as plant_id_eia.'), - sa.Column('subplant_id', sa.Integer(), nullable=True, comment='Sub-plant ID links EPA CEMS emissions units to EIA units.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('emissions_unit_id_epa', sa.Text(), nullable=True, comment='Emissions (smokestack) unit monitored by EPA CEMS.'), - sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!') - ) - op.create_table('ferc_accounts', - sa.Column('ferc_account_id', sa.Text(), nullable=False, comment="Account identifier from FERC's Uniform System of Accounts for Electric Plant. Includes higher level labeled categories."), - sa.Column('ferc_account_description', sa.Text(), nullable=True), - sa.PrimaryKeyConstraint('ferc_account_id', name=op.f('pk_ferc_accounts')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_environmental_equipment_manufacturers')) ) - op.create_table('firing_types_eia', + op.create_table('core_eia__codes_firing_types', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_firing_types_eia')) - ) - op.create_table('fuel_receipts_costs_aggs_eia', - sa.Column('fuel_agg', sa.Text(), nullable=False, comment='Category of fuel aggregation in EIA bulk electricity data.'), - sa.Column('geo_agg', sa.Text(), nullable=False, comment='Category of geographic aggregation in EIA bulk electricity data.'), - sa.Column('sector_agg', sa.Text(), nullable=False, comment='Category of sectoral aggregation in EIA bulk electricity data.'), - sa.Column('temporal_agg', sa.Text(), nullable=False, comment='Category of temporal aggregation in EIA bulk electricity data.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('fuel_received_mmbtu', sa.Float(), nullable=True, comment='Aggregated fuel receipts, in MMBtu, in EIA bulk electricity data.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.PrimaryKeyConstraint('fuel_agg', 'geo_agg', 'sector_agg', 'temporal_agg', 'report_date', name=op.f('pk_fuel_receipts_costs_aggs_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_firing_types')) ) - op.create_table('fuel_transportation_modes_eia', + op.create_table('core_eia__codes_fuel_transportation_modes', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_fuel_transportation_modes_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_fuel_transportation_modes')) ) - op.create_table('fuel_types_aer_eia', + op.create_table('core_eia__codes_fuel_types_aer', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_fuel_types_aer_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_fuel_types_aer')) ) - op.create_table('mercury_compliance_strategies_eia', + op.create_table('core_eia__codes_mercury_compliance_strategies', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_mercury_compliance_strategies_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_mercury_compliance_strategies')) ) - op.create_table('momentary_interruptions_eia', + op.create_table('core_eia__codes_momentary_interruptions', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_momentary_interruptions_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_momentary_interruptions')) ) - op.create_table('nox_compliance_strategies_eia', + op.create_table('core_eia__codes_nox_compliance_strategies', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_nox_compliance_strategies_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_nox_compliance_strategies')) ) - op.create_table('nox_control_status_eia', + op.create_table('core_eia__codes_nox_control_status', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_nox_control_status_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_nox_control_status')) ) - op.create_table('nox_units_eia', + op.create_table('core_eia__codes_nox_units', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_nox_units_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_nox_units')) ) - op.create_table('operational_status_eia', + op.create_table('core_eia__codes_operational_status', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_operational_status_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_operational_status')) ) - op.create_table('particulate_compliance_strategies_eia', + op.create_table('core_eia__codes_particulate_compliance_strategies', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_particulate_compliance_strategies_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_particulate_compliance_strategies')) ) - op.create_table('particulate_units_eia', + op.create_table('core_eia__codes_particulate_units', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_particulate_units_eia')) - ) - op.create_table('plants_entity_eia', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('city', sa.Text(), nullable=True), - sa.Column('county', sa.Text(), nullable=True, comment='County name.'), - sa.Column('latitude', sa.Float(), nullable=True, comment="Latitude of the plant's location, in degrees."), - sa.Column('longitude', sa.Float(), nullable=True, comment="Longitude of the plant's location, in degrees."), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('street_address', sa.Text(), nullable=True), - sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), - sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), - sa.PrimaryKeyConstraint('plant_id_eia', name=op.f('pk_plants_entity_eia')) - ) - op.create_table('plants_pudl', - sa.Column('plant_id_pudl', sa.Integer(), nullable=False, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_pudl', sa.Text(), nullable=True, comment='Plant name, chosen arbitrarily from the several possible plant names available in the plant matching process. Included for human readability only.'), - sa.PrimaryKeyConstraint('plant_id_pudl', name=op.f('pk_plants_pudl')) - ) - op.create_table('political_subdivisions', - sa.Column('country_code', sa.Enum('USA', 'CAN'), nullable=False, comment='Three letter ISO-3166 country code (e.g. USA or CAN).'), - sa.Column('country_name', sa.Text(), nullable=True, comment='Full country name (e.g. United States of America).'), - sa.Column('subdivision_code', sa.Enum('OH', 'NT', 'ID', 'ND', 'MP', 'NM', 'PA', 'IA', 'MD', 'WV', 'AK', 'CT', 'SC', 'TX', 'CA', 'TN', 'UT', 'KY', 'GU', 'MO', 'SD', 'WA', 'NE', 'ON', 'NL', 'KS', 'MA', 'RI', 'WY', 'FL', 'IN', 'MS', 'YT', 'NH', 'OK', 'IL', 'VT', 'SK', 'ME', 'GA', 'PE', 'AS', 'MB', 'AL', 'NC', 'NJ', 'NS', 'PR', 'AR', 'MN', 'OR', 'NV', 'AB', 'MT', 'NB', 'HI', 'DC', 'NY', 'BC', 'CO', 'LA', 'MI', 'NU', 'WI', 'AZ', 'QC', 'VI', 'VA', 'DE'), nullable=False, comment='Two-letter ISO-3166 political subdivision code (e.g. US state or Canadian provice abbreviations like CA or AB).'), - sa.Column('subdivision_name', sa.Text(), nullable=True, comment='Full name of political subdivision (e.g. US state or Canadian province names like California or Alberta.'), - sa.Column('subdivision_type', sa.Text(), nullable=True, comment='ISO-3166 political subdivision type. E.g. state, province, outlying_area.'), - sa.Column('timezone_approx', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name of the timezone which encompasses the largest portion of the population in the associated geographic area.'), - sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), - sa.Column('division_name_us_census', sa.Text(), nullable=True, comment='Longer human readable name describing the US Census division.'), - sa.Column('division_code_us_census', sa.Enum('ENC', 'WNC', 'NEW', 'MTN', 'WSC', 'PCC', 'PCN', 'MAT', 'SAT', 'ESC'), nullable=True, comment='Three-letter US Census division code as it appears in the bulk electricity data published by the EIA. Note that EIA splits the Pacific division into distinct contiguous (CA, OR, WA) and non-contiguous (AK, HI) states. For reference see this US Census region and division map: https://www2.census.gov/geo/pdfs/maps-data/maps/reference/us_regdiv.pdf'), - sa.Column('region_name_us_census', sa.Text(), nullable=True, comment='Human-readable name of a US Census region.'), - sa.Column('is_epacems_state', sa.Boolean(), nullable=True, comment="Indicates whether the associated state reports data within the EPA's Continuous Emissions Monitoring System."), - sa.PrimaryKeyConstraint('country_code', 'subdivision_code', name=op.f('pk_political_subdivisions')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_particulate_units')) ) - op.create_table('power_purchase_types_ferc1', + op.create_table('core_eia__codes_prime_movers', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_power_purchase_types_ferc1')) - ) - op.create_table('predicted_state_hourly_demand', - sa.Column('state_id_fips', sa.Text(), nullable=False, comment='Two digit state FIPS code.'), - sa.Column('utc_datetime', sqlite.DATETIME(), nullable=False), - sa.Column('demand_mwh', sa.Float(), nullable=True), - sa.Column('scaled_demand_mwh', sa.Float(), nullable=True, comment='Estimated electricity demand scaled by the total sales within a state.'), - sa.PrimaryKeyConstraint('state_id_fips', 'utc_datetime', name=op.f('pk_predicted_state_hourly_demand')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_prime_movers')) ) - op.create_table('prime_movers_eia', + op.create_table('core_eia__codes_regulations', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_prime_movers_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_regulations')) ) - op.create_table('regulations_eia', + op.create_table('core_eia__codes_reporting_frequencies', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_regulations_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_reporting_frequencies')) ) - op.create_table('reporting_frequencies_eia', + op.create_table('core_eia__codes_sector_consolidated', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_reporting_frequencies_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_sector_consolidated')) ) - op.create_table('respondent_id_ferc714', - sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False), - sa.Column('respondent_name_ferc714', sa.Text(), nullable=True), - sa.Column('eia_code', sa.Integer(), nullable=True), - sa.PrimaryKeyConstraint('respondent_id_ferc714', name=op.f('pk_respondent_id_ferc714')) - ) - op.create_table('sector_consolidated_eia', - sa.Column('code', sa.Integer(), nullable=False, comment='Originally reported short code.'), + op.create_table('core_eia__codes_so2_compliance_strategies', + sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_sector_consolidated_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_so2_compliance_strategies')) ) - op.create_table('so2_compliance_strategies_eia', + op.create_table('core_eia__codes_so2_units', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_so2_compliance_strategies_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_so2_units')) ) - op.create_table('so2_units_eia', + op.create_table('core_eia__codes_steam_plant_types', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_so2_units_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_steam_plant_types')) ) - op.create_table('steam_plant_types_eia', + op.create_table('core_eia__codes_wet_dry_bottom', sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_steam_plant_types_eia')) + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_eia__codes_wet_dry_bottom')) + ) + op.create_table('core_eia__entity_plants', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('county', sa.Text(), nullable=True, comment='County name.'), + sa.Column('latitude', sa.Float(), nullable=True, comment="Latitude of the plant's location, in degrees."), + sa.Column('longitude', sa.Float(), nullable=True, comment="Longitude of the plant's location, in degrees."), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), + sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), + sa.PrimaryKeyConstraint('plant_id_eia', name=op.f('pk_core_eia__entity_plants')) ) - op.create_table('utilities_entity_eia', + op.create_table('core_eia__entity_utilities', sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.PrimaryKeyConstraint('utility_id_eia', name=op.f('pk_utilities_entity_eia')) + sa.PrimaryKeyConstraint('utility_id_eia', name=op.f('pk_core_eia__entity_utilities')) + ) + op.create_table('core_eia__yearly_fuel_receipts_costs_aggs', + sa.Column('fuel_agg', sa.Text(), nullable=False, comment='Category of fuel aggregation in EIA bulk electricity data.'), + sa.Column('geo_agg', sa.Text(), nullable=False, comment='Category of geographic aggregation in EIA bulk electricity data.'), + sa.Column('sector_agg', sa.Text(), nullable=False, comment='Category of sectoral aggregation in EIA bulk electricity data.'), + sa.Column('temporal_agg', sa.Text(), nullable=False, comment='Category of temporal aggregation in EIA bulk electricity data.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('fuel_received_mmbtu', sa.Float(), nullable=True, comment='Aggregated fuel receipts, in MMBtu, in EIA bulk electricity data.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.PrimaryKeyConstraint('fuel_agg', 'geo_agg', 'sector_agg', 'temporal_agg', 'report_date', name=op.f('pk_core_eia__yearly_fuel_receipts_costs_aggs')) + ) + op.create_table('core_epa__assn_eia_epacamd_subplant_ids', + sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_epa', sa.Integer(), nullable=True, comment='The ORISPL ID used by EPA to refer to the plant. Usually but not always the same as plant_id_eia.'), + sa.Column('subplant_id', sa.Integer(), nullable=True, comment='Sub-plant ID links EPA CEMS emissions units to EIA units.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('emissions_unit_id_epa', sa.Text(), nullable=True, comment='Emissions (smokestack) unit monitored by EPA CEMS.'), + sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!') + ) + op.create_table('core_ferc1__codes_power_purchase_types', + sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), + sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), + sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_ferc1__codes_power_purchase_types')) + ) + op.create_table('core_ferc714__respondent_id', + sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False, comment='FERC Form 714 respondent ID. Note that this ID does not correspond to FERC respondent IDs from other forms.'), + sa.Column('respondent_name_ferc714', sa.Text(), nullable=True, comment='Name of the utility, balancing area authority, or planning authority responding to FERC Form 714.'), + sa.Column('eia_code', sa.Integer(), nullable=True, comment='EIA utility or balancing area authority ID associated with this FERC Form 714 respondent. Note that many utilities are also balancing authorities and in many cases EIA uses the same integer ID to identify a utility in its role as a balancing authority AND as a utility, but there is no requirement that these IDs be the same, and in a number of cases they are different.'), + sa.PrimaryKeyConstraint('respondent_id_ferc714', name=op.f('pk_core_ferc714__respondent_id')) + ) + op.create_table('core_ferc__codes_accounts', + sa.Column('ferc_account_id', sa.Text(), nullable=False, comment="Account identifier from FERC's Uniform System of Accounts for Electric Plant. Includes higher level labeled categories."), + sa.Column('ferc_account_description', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('ferc_account_id', name=op.f('pk_core_ferc__codes_accounts')) + ) + op.create_table('core_pudl__codes_data_maturities', + sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), + sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), + sa.PrimaryKeyConstraint('code', name=op.f('pk_core_pudl__codes_data_maturities')) + ) + op.create_table('core_pudl__codes_datasources', + sa.Column('datasource', sa.Enum('censusdp1tract', 'eia176', 'eia860', 'eia860m', 'eia861', 'eia923', 'eia_bulk_elec', 'eiawater', 'epacems', 'core_epa__assn_eia_epacamd', 'ferc1', 'ferc2', 'ferc6', 'ferc60', 'ferc714', 'ferceqr', 'mshamines', 'phmsagas', 'pudl'), nullable=False, comment='Code identifying a dataset available within PUDL.'), + sa.Column('partitions', sa.Text(), nullable=True, comment='The data parititions used to generate this instance of the database.'), + sa.Column('doi', sa.Text(), nullable=True, comment='Unique digitial object identifier of Zenodo archive.'), + sa.Column('pudl_version', sa.Text(), nullable=True, comment='The version of PUDL used to generate this database.'), + sa.PrimaryKeyConstraint('datasource', name=op.f('pk_core_pudl__codes_datasources')) + ) + op.create_table('core_pudl__codes_subdivisions', + sa.Column('country_code', sa.Enum('USA', 'CAN'), nullable=False, comment='Three letter ISO-3166 country code (e.g. USA or CAN).'), + sa.Column('country_name', sa.Text(), nullable=True, comment='Full country name (e.g. United States of America).'), + sa.Column('subdivision_code', sa.Enum('NJ', 'KY', 'PE', 'NT', 'DE', 'MP', 'ON', 'MB', 'OK', 'SD', 'GU', 'CA', 'CT', 'NE', 'AS', 'UT', 'KS', 'MO', 'NC', 'QC', 'NB', 'AL', 'AR', 'BC', 'NU', 'OR', 'VT', 'NM', 'PR', 'WI', 'WA', 'PA', 'RI', 'TX', 'GA', 'MA', 'AB', 'AZ', 'YT', 'ME', 'VI', 'SC', 'IN', 'WV', 'FL', 'NY', 'MS', 'ID', 'DC', 'IL', 'MI', 'CO', 'LA', 'NL', 'MD', 'OH', 'NS', 'AK', 'SK', 'VA', 'MT', 'ND', 'NH', 'MN', 'TN', 'IA', 'HI', 'WY', 'NV'), nullable=False, comment='Two-letter ISO-3166 political subdivision code (e.g. US state or Canadian provice abbreviations like CA or AB).'), + sa.Column('subdivision_name', sa.Text(), nullable=True, comment='Full name of political subdivision (e.g. US state or Canadian province names like California or Alberta.'), + sa.Column('subdivision_type', sa.Text(), nullable=True, comment='ISO-3166 political subdivision type. E.g. state, province, outlying_area.'), + sa.Column('timezone_approx', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name of the timezone which encompasses the largest portion of the population in the associated geographic area.'), + sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), + sa.Column('division_name_us_census', sa.Text(), nullable=True, comment='Longer human readable name describing the US Census division.'), + sa.Column('division_code_us_census', sa.Enum('SAT', 'PCC', 'WNC', 'WSC', 'MTN', 'NEW', 'ENC', 'MAT', 'ESC', 'PCN'), nullable=True, comment='Three-letter US Census division code as it appears in the bulk electricity data published by the EIA. Note that EIA splits the Pacific division into distinct contiguous (CA, OR, WA) and non-contiguous (AK, HI) states. For reference see this US Census region and division map: https://www2.census.gov/geo/pdfs/maps-data/maps/reference/us_regdiv.pdf'), + sa.Column('region_name_us_census', sa.Text(), nullable=True, comment='Human-readable name of a US Census region.'), + sa.Column('is_epacems_state', sa.Boolean(), nullable=True, comment="Indicates whether the associated state reports data within the EPA's Continuous Emissions Monitoring System."), + sa.PrimaryKeyConstraint('country_code', 'subdivision_code', name=op.f('pk_core_pudl__codes_subdivisions')) + ) + op.create_table('core_pudl__entity_plants_pudl', + sa.Column('plant_id_pudl', sa.Integer(), nullable=False, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_pudl', sa.Text(), nullable=True, comment='Plant name, chosen arbitrarily from the several possible plant names available in the plant matching process. Included for human readability only.'), + sa.PrimaryKeyConstraint('plant_id_pudl', name=op.f('pk_core_pudl__entity_plants_pudl')) ) - op.create_table('utilities_pudl', + op.create_table('core_pudl__entity_utilities_pudl', sa.Column('utility_id_pudl', sa.Integer(), nullable=False, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_pudl', sa.Text(), nullable=True, comment='Utility name, chosen arbitrarily from the several possible utility names available in the utility matching process. Included for human readability only.'), - sa.PrimaryKeyConstraint('utility_id_pudl', name=op.f('pk_utilities_pudl')) + sa.PrimaryKeyConstraint('utility_id_pudl', name=op.f('pk_core_pudl__entity_utilities_pudl')) + ) + op.create_table('out_eia861__compiled_geometry_balancing_authorities', + sa.Column('county_id_fips', sa.Text(), nullable=False, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), + sa.Column('county_name_census', sa.Text(), nullable=True, comment='County name as specified in Census DP1 Data.'), + sa.Column('population', sa.Float(), nullable=True, comment='County population, sourced from Census DP1 data.'), + sa.Column('area_km2', sa.Float(), nullable=True, comment='County area in km2.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=False, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('county', sa.Text(), nullable=False, comment='County name.'), + sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), + sa.PrimaryKeyConstraint('balancing_authority_id_eia', 'report_date', 'county_id_fips', 'county', name=op.f('pk_out_eia861__compiled_geometry_balancing_authorities')) ) - op.create_table('utility_assn_eia861', + op.create_table('out_eia861__compiled_geometry_utilities', + sa.Column('county_id_fips', sa.Text(), nullable=False, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), + sa.Column('county_name_census', sa.Text(), nullable=True, comment='County name as specified in Census DP1 Data.'), + sa.Column('population', sa.Float(), nullable=True, comment='County population, sourced from Census DP1 data.'), + sa.Column('area_km2', sa.Float(), nullable=True, comment='County area in km2.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), - sa.Column('state', sa.Text(), nullable=False, comment='Two letter US state abbreviation.'), - sa.PrimaryKeyConstraint('report_date', 'utility_id_eia', 'state', name=op.f('pk_utility_assn_eia861')) + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('county', sa.Text(), nullable=True, comment='County name.'), + sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), + sa.PrimaryKeyConstraint('utility_id_eia', 'report_date', 'county_id_fips', name=op.f('pk_out_eia861__compiled_geometry_utilities')) ) - op.create_table('wet_dry_bottom_eia', - sa.Column('code', sa.Text(), nullable=False, comment='Originally reported short code.'), - sa.Column('label', sa.Text(), nullable=True, comment='Longer human-readable code using snake_case'), - sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), - sa.PrimaryKeyConstraint('code', name=op.f('pk_wet_dry_bottom_eia')) + op.create_table('out_ferc714__hourly_predicted_state_demand', + sa.Column('state_id_fips', sa.Text(), nullable=False, comment='Two digit state FIPS code.'), + sa.Column('utc_datetime', sqlite.DATETIME(), nullable=False), + sa.Column('demand_mwh', sa.Float(), nullable=True), + sa.Column('scaled_demand_mwh', sa.Float(), nullable=True, comment='Estimated electricity demand scaled by the total sales within a state.'), + sa.PrimaryKeyConstraint('state_id_fips', 'utc_datetime', name=op.f('pk_out_ferc714__hourly_predicted_state_demand')) + ) + op.create_table('_out_eia__monthly_heat_rate_by_unit', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=False, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk__out_eia__monthly_heat_rate_by_unit_plant_id_eia_core_eia__entity_plants')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'unit_id_pudl', name=op.f('pk__out_eia__monthly_heat_rate_by_unit')) + ) + op.create_table('core_eia860__scd_emissions_control_equipment', + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('emission_control_id_pudl', sa.Float(), nullable=False, comment='A PUDL-generated ID used to distinguish emission control units in the same report year and plant id. This ID should not be used to track units over time or between plants.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.Column('emission_control_equipment_type_code', sa.Text(), nullable=True, comment='Short code indicating the type of emission control equipment installed.'), + sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), + sa.Column('mercury_control_id_eia', sa.Text(), nullable=True, comment='Mercury control identification number. This ID is not a unique identifier.'), + sa.Column('nox_control_id_eia', sa.Text(), nullable=True, comment='Nitrogen oxide control identification number. This ID is not a unique identifier.'), + sa.Column('particulate_control_id_eia', sa.Text(), nullable=True, comment='Particulate matter control identification number. This ID is not a unique identifier.'), + sa.Column('so2_control_id_eia', sa.Text(), nullable=True, comment='Sulfur dioxide control identification number. This ID is not a unique identifier.'), + sa.Column('acid_gas_control', sa.Boolean(), nullable=True, comment='Indicates whether the emissions control equipment controls acid (HCl) gas.'), + sa.Column('emission_control_equipment_cost', sa.Float(), nullable=True, comment='The total cost to install a piece of emission control equipment.'), + sa.Column('emission_control_operating_date', sa.Date(), nullable=True, comment='The date a piece of emissions control equipment began operating. Derived from month and year columns in the raw data.'), + sa.Column('emission_control_retirement_date', sa.Date(), nullable=True, comment='The expected or actual retirement date for a piece of emissions control equipment. Derived from month and year columns in the raw data.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__scd_emissions_control_equipment_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['emission_control_equipment_type_code'], ['core_eia__codes_emission_control_equipment_types.code'], name=op.f('fk_core_eia860__scd_emissions_control_equipment_emission_control_equipment_type_code_core_eia__codes_emission_control_equipment_types')), + sa.ForeignKeyConstraint(['operational_status_code'], ['core_eia__codes_operational_status.code'], name=op.f('fk_core_eia860__scd_emissions_control_equipment_operational_status_code_core_eia__codes_operational_status')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia860__scd_emissions_control_equipment_plant_id_eia_core_eia__entity_plants')), + sa.PrimaryKeyConstraint('report_year', 'plant_id_eia', 'emission_control_id_pudl', name=op.f('pk_core_eia860__scd_emissions_control_equipment')) + ) + op.create_table('core_eia860__scd_utilities', + sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), + sa.Column('plants_reported_owner', sa.Boolean(), nullable=True, comment='Is the reporting entity an owner of power plants reported on Schedule 2 of the form?'), + sa.Column('plants_reported_operator', sa.Boolean(), nullable=True, comment='Is the reporting entity an operator of power plants reported on Schedule 2 of the form?'), + sa.Column('plants_reported_asset_manager', sa.Boolean(), nullable=True, comment='Is the reporting entity an asset manager of power plants reported on Schedule 2 of the form?'), + sa.Column('plants_reported_other_relationship', sa.Boolean(), nullable=True, comment='Does the reporting entity have any other relationship to the power plants reported on Schedule 2 of the form?'), + sa.Column('entity_type', sa.Text(), nullable=True, comment='Entity type of principal owner.'), + sa.Column('attention_line', sa.Text(), nullable=True), + sa.Column('address_2', sa.Text(), nullable=True), + sa.Column('zip_code_4', sa.Text(), nullable=True, comment='Four digit US Zip Code suffix.'), + sa.Column('contact_firstname', sa.Text(), nullable=True, comment='First name of utility contact 1.'), + sa.Column('contact_lastname', sa.Text(), nullable=True, comment='Last name of utility contact 1.'), + sa.Column('contact_title', sa.Text(), nullable=True, comment='Title of of utility contact 1.'), + sa.Column('phone_number', sa.Text(), nullable=True, comment='Phone number for utility contact 1.'), + sa.Column('phone_extension', sa.Text(), nullable=True, comment='Phone extension for utility contact 1'), + sa.Column('contact_firstname_2', sa.Text(), nullable=True, comment='First name of utility contact 2.'), + sa.Column('contact_lastname_2', sa.Text(), nullable=True, comment='Last name of utility contact 2.'), + sa.Column('contact_title_2', sa.Text(), nullable=True, comment='Title of utility contact 2.'), + sa.Column('phone_number_2', sa.Text(), nullable=True, comment='Phone number for utility contact 2.'), + sa.Column('phone_extension_2', sa.Text(), nullable=True, comment='Phone extension for utility contact 2'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__scd_utilities_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_core_eia860__scd_utilities_utility_id_eia_core_eia__entity_utilities')), + sa.PrimaryKeyConstraint('utility_id_eia', 'report_date', name=op.f('pk_core_eia860__scd_utilities')) ) - op.create_table('advanced_metering_infrastructure_eia861', + op.create_table('core_eia861__yearly_advanced_metering_infrastructure', sa.Column('advanced_metering_infrastructure', sa.Integer(), nullable=True), sa.Column('automated_meter_reading', sa.Integer(), nullable=True), sa.Column('balancing_authority_code_eia', sa.Text(), nullable=False, comment='EIA short code identifying a balancing authority.'), @@ -356,40 +417,10 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_advanced_metering_infrastructure_eia861_data_maturity_data_maturities')), - sa.PrimaryKeyConstraint('balancing_authority_code_eia', 'customer_class', 'report_date', 'state', 'utility_id_eia', name=op.f('pk_advanced_metering_infrastructure_eia861')) - ) - op.create_table('boilers_entity_eia', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), - sa.Column('boiler_manufacturer', sa.Text(), nullable=True, comment='Name of boiler manufacturer.'), - sa.Column('boiler_manufacturer_code', sa.Text(), nullable=True, comment='EIA short code for boiler manufacturer.'), - sa.ForeignKeyConstraint(['boiler_manufacturer_code'], ['environmental_equipment_manufacturers_eia.code'], name=op.f('fk_boilers_entity_eia_boiler_manufacturer_code_environmental_equipment_manufacturers_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_boilers_entity_eia_plant_id_eia_plants_entity_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', name=op.f('pk_boilers_entity_eia')) - ) - op.create_table('coalmine_eia923', - sa.Column('mine_id_pudl', sa.Integer(), nullable=False, comment='Dynamically assigned PUDL mine identifier.'), - sa.Column('mine_name', sa.Text(), nullable=True, comment='Coal mine name.'), - sa.Column('mine_type_code', sa.Text(), nullable=True, comment='Type of coal mine.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('county_id_fips', sa.Text(), nullable=True, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), - sa.Column('mine_id_msha', sa.Integer(), nullable=True, comment='MSHA issued mine identifier.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_coalmine_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['mine_type_code'], ['coalmine_types_eia.code'], name=op.f('fk_coalmine_eia923_mine_type_code_coalmine_types_eia')), - sa.PrimaryKeyConstraint('mine_id_pudl', name=op.f('pk_coalmine_eia923')) - ) - op.create_table('demand_hourly_pa_ferc714', - sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False), - sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('utc_datetime', sqlite.DATETIME(), nullable=False), - sa.Column('timezone', sa.Enum('America/New_York', 'America/Chicago', 'America/Denver', 'America/Los_Angeles', 'America/Anchorage', 'Pacific/Honolulu'), nullable=True, comment='IANA timezone name'), - sa.Column('demand_mwh', sa.Float(), nullable=True), - sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['respondent_id_ferc714.respondent_id_ferc714'], name=op.f('fk_demand_hourly_pa_ferc714_respondent_id_ferc714_respondent_id_ferc714')), - sa.PrimaryKeyConstraint('respondent_id_ferc714', 'utc_datetime', name=op.f('pk_demand_hourly_pa_ferc714')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_advanced_metering_infrastructure_data_maturity_core_pudl__codes_data_maturities')), + sa.PrimaryKeyConstraint('balancing_authority_code_eia', 'customer_class', 'report_date', 'state', 'utility_id_eia', name=op.f('pk_core_eia861__yearly_advanced_metering_infrastructure')) ) - op.create_table('demand_response_eia861', + op.create_table('core_eia861__yearly_demand_response', sa.Column('actual_peak_demand_savings_mw', sa.Float(), nullable=True), sa.Column('balancing_authority_code_eia', sa.Text(), nullable=False, comment='EIA short code identifying a balancing authority.'), sa.Column('customer_class', sa.Enum('commercial', 'industrial', 'direct_connection', 'other', 'residential', 'total', 'transportation'), nullable=False, comment='High level categorization of customer type.'), @@ -404,20 +435,20 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_demand_response_eia861_data_maturity_data_maturities')), - sa.PrimaryKeyConstraint('balancing_authority_code_eia', 'customer_class', 'report_date', 'state', 'utility_id_eia', name=op.f('pk_demand_response_eia861')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_demand_response_data_maturity_core_pudl__codes_data_maturities')), + sa.PrimaryKeyConstraint('balancing_authority_code_eia', 'customer_class', 'report_date', 'state', 'utility_id_eia', name=op.f('pk_core_eia861__yearly_demand_response')) ) - op.create_table('demand_response_water_heater_eia861', + op.create_table('core_eia861__yearly_demand_response_water_heater', sa.Column('balancing_authority_code_eia', sa.Text(), nullable=False, comment='EIA short code identifying a balancing authority.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('state', sa.Text(), nullable=False, comment='Two letter US state abbreviation.'), sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('water_heater', sa.Integer(), nullable=True), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_demand_response_water_heater_eia861_data_maturity_data_maturities')), - sa.PrimaryKeyConstraint('balancing_authority_code_eia', 'report_date', 'state', 'utility_id_eia', name=op.f('pk_demand_response_water_heater_eia861')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_demand_response_water_heater_data_maturity_core_pudl__codes_data_maturities')), + sa.PrimaryKeyConstraint('balancing_authority_code_eia', 'report_date', 'state', 'utility_id_eia', name=op.f('pk_core_eia861__yearly_demand_response_water_heater')) ) - op.create_table('demand_side_management_ee_dr_eia861', + op.create_table('core_eia861__yearly_demand_side_management_ee_dr', sa.Column('annual_indirect_program_cost', sa.Float(), nullable=True), sa.Column('annual_total_cost', sa.Float(), nullable=True), sa.Column('customer_class', sa.Enum('commercial', 'industrial', 'direct_connection', 'other', 'residential', 'total', 'transportation'), nullable=True, comment='High level categorization of customer type.'), @@ -442,9 +473,9 @@ def upgrade() -> None: sa.Column('time_responsiveness_customers', sa.Integer(), nullable=True), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_demand_side_management_ee_dr_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_demand_side_management_ee_dr_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('demand_side_management_misc_eia861', + op.create_table('core_eia861__yearly_demand_side_management_misc', sa.Column('energy_savings_estimates_independently_verified', sa.Boolean(), nullable=True), sa.Column('energy_savings_independently_verified', sa.Boolean(), nullable=True), sa.Column('entity_type', sa.Text(), nullable=True, comment='Entity type of principal owner.'), @@ -459,9 +490,9 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_demand_side_management_misc_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_demand_side_management_misc_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('demand_side_management_sales_eia861', + op.create_table('core_eia861__yearly_demand_side_management_sales', sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), sa.Column('sales_for_resale_mwh', sa.Float(), nullable=True), @@ -469,191 +500,33 @@ def upgrade() -> None: sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_demand_side_management_sales_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_demand_side_management_sales_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('denorm_emissions_control_equipment_eia860', - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + op.create_table('core_eia861__yearly_distributed_generation_fuel', + sa.Column('estimated_or_actual_fuel_data', sa.Enum('estimated', 'actual'), nullable=True), + sa.Column('fuel_class', sa.Enum('gas', 'oil', 'other', 'renewable', 'water', 'wind', 'wood'), nullable=True), + sa.Column('fuel_pct', sa.Float(), nullable=True), + sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_distributed_generation_fuel_data_maturity_core_pudl__codes_data_maturities')) + ) + op.create_table('core_eia861__yearly_distributed_generation_misc', + sa.Column('backup_capacity_mw', sa.Float(), nullable=True), + sa.Column('distributed_generation_owned_capacity_mw', sa.Float(), nullable=True), + sa.Column('estimated_or_actual_capacity_data', sa.Enum('estimated', 'actual'), nullable=True), + sa.Column('generators_num_less_1_mw', sa.Float(), nullable=True), + sa.Column('generators_number', sa.Float(), nullable=True), + sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('total_capacity_less_1_mw', sa.Float(), nullable=True), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('emission_control_id_pudl', sa.Float(), nullable=False, comment='A PUDL-generated ID used to distinguish emission control units in the same report year and plant id. This ID should not be used to track units over time or between plants.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.Column('emission_control_equipment_type_code', sa.Text(), nullable=True, comment='Short code indicating the type of emission control equipment installed.'), - sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), - sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), - sa.Column('mercury_control_id_eia', sa.Text(), nullable=True, comment='Mercury control identification number. This ID is not a unique identifier.'), - sa.Column('nox_control_id_eia', sa.Text(), nullable=True, comment='Nitrogen oxide control identification number. This ID is not a unique identifier.'), - sa.Column('particulate_control_id_eia', sa.Text(), nullable=True, comment='Particulate matter control identification number. This ID is not a unique identifier.'), - sa.Column('so2_control_id_eia', sa.Text(), nullable=True, comment='Sulfur dioxide control identification number. This ID is not a unique identifier.'), - sa.Column('acid_gas_control', sa.Boolean(), nullable=True, comment='Indicates whether the emissions control equipment controls acid (HCl) gas.'), - sa.Column('emission_control_equipment_cost', sa.Float(), nullable=True, comment='The total cost to install a piece of emission control equipment.'), - sa.Column('emission_control_operating_date', sa.Date(), nullable=True, comment='The date a piece of emissions control equipment began operating. Derived from month and year columns in the raw data.'), - sa.Column('emission_control_retirement_date', sa.Date(), nullable=True, comment='The expected or actual retirement date for a piece of emissions control equipment. Derived from month and year columns in the raw data.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_emissions_control_equipment_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['emission_control_equipment_type_code'], ['emission_control_equipment_types_eia.code'], name=op.f('fk_denorm_emissions_control_equipment_eia860_emission_control_equipment_type_code_emission_control_equipment_types_eia')), - sa.ForeignKeyConstraint(['operational_status_code'], ['operational_status_eia.code'], name=op.f('fk_denorm_emissions_control_equipment_eia860_operational_status_code_operational_status_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_denorm_emissions_control_equipment_eia860_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_emissions_control_equipment_eia860_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_emissions_control_equipment_eia860_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_emissions_control_equipment_eia860_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_year', 'plant_id_eia', 'emission_control_id_pudl', name=op.f('pk_denorm_emissions_control_equipment_eia860')) - ) - op.create_table('denorm_fuel_receipts_costs_eia923', - sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('contract_type_code', sa.Enum('S', 'C', 'NC', 'T'), nullable=True, comment='Purchase type under which receipts occurred in the reporting month. C: Contract, NC: New Contract, S: Spot Purchase, T: Tolling Agreement.'), - sa.Column('contract_expiration_date', sa.Date(), nullable=True, comment='Date contract expires.Format: MMYY.'), - sa.Column('energy_source_code', sa.Text(), nullable=True, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_group_code', sa.Enum('petroleum', 'other_gas', 'petroleum_coke', 'natural_gas', 'coal'), nullable=True, comment='Fuel groups used in the Electric Power Monthly'), - sa.Column('supplier_name', sa.Text(), nullable=True, comment='Company that sold the fuel to the plant or, in the case of Natural Gas, pipline owner.'), - sa.Column('fuel_received_units', sa.Float(), nullable=True, comment='Quanity of fuel received in tons, barrel, or Mcf.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('bulk_agg_fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Fuel cost per mmbtu reported in the EIA bulk electricity data. This is an aggregate average fuel price for a whole state, region, month, sector, etc. Used to fill in missing fuel prices.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), - sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), - sa.Column('mercury_content_ppm', sa.Float(), nullable=True, comment='Mercury content in parts per million (ppm) to the nearest 0.001 ppm.'), - sa.Column('primary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the longest distance transported.'), - sa.Column('secondary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the second longest distance transported.'), - sa.Column('natural_gas_transport_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natural gas transportation service.'), - sa.Column('natural_gas_delivery_contract_type_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natrual gas delivery service:'), - sa.Column('moisture_content_pct', sa.Float(), nullable=True), - sa.Column('chlorine_content_ppm', sa.Float(), nullable=True), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.Column('mine_id_msha', sa.Integer(), nullable=True, comment='MSHA issued mine identifier.'), - sa.Column('mine_name', sa.Text(), nullable=True, comment='Coal mine name.'), - sa.Column('mine_state', sa.Text(), nullable=True, comment='State where the coal mine is located. Two letter abbreviation.'), - sa.Column('coalmine_county_id_fips', sa.Text(), nullable=True, comment='County ID from the Federal Information Processing Standard Publication 6-4. This is the county where the coal mine is located.'), - sa.Column('mine_type_code', sa.Text(), nullable=True, comment='Type of coal mine.'), - sa.ForeignKeyConstraint(['contract_type_code'], ['contract_types_eia.code'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_contract_type_code_contract_types_eia')), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['mine_type_code'], ['coalmine_types_eia.code'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_mine_type_code_coalmine_types_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['primary_transportation_mode_code'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_primary_transportation_mode_code_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['secondary_transportation_mode_code'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_secondary_transportation_mode_code_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_fuel_receipts_costs_eia923_utility_id_pudl_utilities_pudl')) - ) - op.create_table('denorm_fuel_receipts_costs_monthly_eia923', - sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_received_units', sa.Float(), nullable=True, comment='Quanity of fuel received in tons, barrel, or Mcf.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), - sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), - sa.Column('mercury_content_ppm', sa.Float(), nullable=True, comment='Mercury content in parts per million (ppm) to the nearest 0.001 ppm.'), - sa.Column('moisture_content_pct', sa.Float(), nullable=True), - sa.Column('chlorine_content_ppm', sa.Float(), nullable=True), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_denorm_fuel_receipts_costs_monthly_eia923_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_fuel_receipts_costs_monthly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_fuel_receipts_costs_monthly_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_fuel_receipts_costs_monthly_eia923_utility_id_pudl_utilities_pudl')) - ) - op.create_table('denorm_generation_fuel_combined_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_type_code_aer', sa.Text(), nullable=True, comment='A partial aggregation of the reported fuel type codes into larger categories used by EIA in, for example, the Annual Energy Review (AER). Two or three letter alphanumeric.'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_generation_fuel_combined_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generation_fuel_combined_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['fuel_type_code_aer'], ['fuel_types_aer_eia.code'], name=op.f('fk_denorm_generation_fuel_combined_eia923_fuel_type_code_aer_fuel_types_aer_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_denorm_generation_fuel_combined_eia923_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_generation_fuel_combined_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_generation_fuel_combined_eia923_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_generation_fuel_combined_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_generation_fuel_combined_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_denorm_generation_fuel_combined_eia923')) - ) - op.create_table('denorm_generation_fuel_combined_monthly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generation_fuel_combined_monthly_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_denorm_generation_fuel_combined_monthly_eia923_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_generation_fuel_combined_monthly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_generation_fuel_combined_monthly_eia923_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_generation_fuel_combined_monthly_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_generation_fuel_combined_monthly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_denorm_generation_fuel_combined_monthly_eia923')) - ) - op.create_table('distributed_generation_fuel_eia861', - sa.Column('estimated_or_actual_fuel_data', sa.Enum('estimated', 'actual'), nullable=True), - sa.Column('fuel_class', sa.Enum('gas', 'oil', 'other', 'renewable', 'water', 'wind', 'wood'), nullable=True), - sa.Column('fuel_pct', sa.Float(), nullable=True), - sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_distributed_generation_fuel_eia861_data_maturity_data_maturities')) - ) - op.create_table('distributed_generation_misc_eia861', - sa.Column('backup_capacity_mw', sa.Float(), nullable=True), - sa.Column('distributed_generation_owned_capacity_mw', sa.Float(), nullable=True), - sa.Column('estimated_or_actual_capacity_data', sa.Enum('estimated', 'actual'), nullable=True), - sa.Column('generators_num_less_1_mw', sa.Float(), nullable=True), - sa.Column('generators_number', sa.Float(), nullable=True), - sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('total_capacity_less_1_mw', sa.Float(), nullable=True), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_distributed_generation_misc_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_distributed_generation_misc_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('distributed_generation_tech_eia861', + op.create_table('core_eia861__yearly_distributed_generation_tech', sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), sa.Column('estimated_or_actual_tech_data', sa.Enum('estimated', 'actual'), nullable=True), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), @@ -661,9 +534,9 @@ def upgrade() -> None: sa.Column('tech_class', sa.Enum('backup', 'chp_cogen', 'combustion_turbine', 'fuel_cell', 'hydro', 'internal_combustion', 'other', 'pv', 'steam', 'storage_pv', 'all_storage', 'total', 'virtual_pv', 'wind'), nullable=True), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_distributed_generation_tech_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_distributed_generation_tech_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('distribution_systems_eia861', + op.create_table('core_eia861__yearly_distribution_systems', sa.Column('circuits_with_voltage_optimization', sa.Integer(), nullable=True), sa.Column('distribution_circuits', sa.Integer(), nullable=True), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), @@ -672,9 +545,9 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_distribution_systems_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_distribution_systems_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('dynamic_pricing_eia861', + op.create_table('core_eia861__yearly_dynamic_pricing', sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), sa.Column('critical_peak_pricing', sa.Boolean(), nullable=True), sa.Column('critical_peak_rebate', sa.Boolean(), nullable=True), @@ -689,30 +562,9 @@ def upgrade() -> None: sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('variable_peak_pricing', sa.Boolean(), nullable=True), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_dynamic_pricing_eia861_data_maturity_data_maturities')) - ) - op.create_table('emissions_control_equipment_eia860', - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('emission_control_id_pudl', sa.Float(), nullable=False, comment='A PUDL-generated ID used to distinguish emission control units in the same report year and plant id. This ID should not be used to track units over time or between plants.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.Column('emission_control_equipment_type_code', sa.Text(), nullable=True, comment='Short code indicating the type of emission control equipment installed.'), - sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), - sa.Column('mercury_control_id_eia', sa.Text(), nullable=True, comment='Mercury control identification number. This ID is not a unique identifier.'), - sa.Column('nox_control_id_eia', sa.Text(), nullable=True, comment='Nitrogen oxide control identification number. This ID is not a unique identifier.'), - sa.Column('particulate_control_id_eia', sa.Text(), nullable=True, comment='Particulate matter control identification number. This ID is not a unique identifier.'), - sa.Column('so2_control_id_eia', sa.Text(), nullable=True, comment='Sulfur dioxide control identification number. This ID is not a unique identifier.'), - sa.Column('acid_gas_control', sa.Boolean(), nullable=True, comment='Indicates whether the emissions control equipment controls acid (HCl) gas.'), - sa.Column('emission_control_equipment_cost', sa.Float(), nullable=True, comment='The total cost to install a piece of emission control equipment.'), - sa.Column('emission_control_operating_date', sa.Date(), nullable=True, comment='The date a piece of emissions control equipment began operating. Derived from month and year columns in the raw data.'), - sa.Column('emission_control_retirement_date', sa.Date(), nullable=True, comment='The expected or actual retirement date for a piece of emissions control equipment. Derived from month and year columns in the raw data.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_emissions_control_equipment_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['emission_control_equipment_type_code'], ['emission_control_equipment_types_eia.code'], name=op.f('fk_emissions_control_equipment_eia860_emission_control_equipment_type_code_emission_control_equipment_types_eia')), - sa.ForeignKeyConstraint(['operational_status_code'], ['operational_status_eia.code'], name=op.f('fk_emissions_control_equipment_eia860_operational_status_code_operational_status_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_emissions_control_equipment_eia860_plant_id_eia_plants_entity_eia')), - sa.PrimaryKeyConstraint('report_year', 'plant_id_eia', 'emission_control_id_pudl', name=op.f('pk_emissions_control_equipment_eia860')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_dynamic_pricing_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('energy_efficiency_eia861', + op.create_table('core_eia861__yearly_energy_efficiency', sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), sa.Column('customer_class', sa.Enum('commercial', 'industrial', 'direct_connection', 'other', 'residential', 'total', 'transportation'), nullable=True, comment='High level categorization of customer type.'), sa.Column('customer_incentives_incremental_cost', sa.Float(), nullable=True), @@ -730,93 +582,9 @@ def upgrade() -> None: sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('weighted_average_life_years', sa.Float(), nullable=True), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_energy_efficiency_eia861_data_maturity_data_maturities')) - ) - op.create_table('fipsified_respondents_ferc714', - sa.Column('eia_code', sa.Integer(), nullable=True), - sa.Column('respondent_type', sa.Enum('utility', 'balancing_authority'), nullable=True), - sa.Column('respondent_id_ferc714', sa.Integer(), nullable=True), - sa.Column('respondent_name_ferc714', sa.Text(), nullable=True), - sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=True, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), - sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), - sa.Column('balancing_authority_name_eia', sa.Text(), nullable=True, comment='Name of the balancing authority.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('county', sa.Text(), nullable=True, comment='County name.'), - sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), - sa.Column('county_id_fips', sa.Text(), nullable=True, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), - sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['respondent_id_ferc714.respondent_id_ferc714'], name=op.f('fk_fipsified_respondents_ferc714_respondent_id_ferc714_respondent_id_ferc714')) - ) - op.create_table('generation_fuel_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_type_code_aer', sa.Text(), nullable=True, comment='A partial aggregation of the reported fuel type codes into larger categories used by EIA in, for example, the Annual Energy Review (AER). Two or three letter alphanumeric.'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_generation_fuel_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_generation_fuel_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['fuel_type_code_aer'], ['fuel_types_aer_eia.code'], name=op.f('fk_generation_fuel_eia923_fuel_type_code_aer_fuel_types_aer_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_generation_fuel_eia923_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_generation_fuel_eia923_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_generation_fuel_eia923')) - ) - op.create_table('generation_fuel_nuclear_eia923', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('nuclear_unit_id', sa.Text(), nullable=False, comment='For nuclear plants only, the unit number .One digit numeric. Nuclear plants are the only type of plants for which data are shown explicitly at the generating unit level.'), - sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_type_code_aer', sa.Text(), nullable=True, comment='A partial aggregation of the reported fuel type codes into larger categories used by EIA in, for example, the Annual Energy Review (AER). Two or three letter alphanumeric.'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_generation_fuel_nuclear_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_generation_fuel_nuclear_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['fuel_type_code_aer'], ['fuel_types_aer_eia.code'], name=op.f('fk_generation_fuel_nuclear_eia923_fuel_type_code_aer_fuel_types_aer_eia')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_generation_fuel_nuclear_eia923_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_generation_fuel_nuclear_eia923_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'nuclear_unit_id', 'energy_source_code', 'prime_mover_code', name=op.f('pk_generation_fuel_nuclear_eia923')) - ) - op.create_table('generators_entity_eia', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('duct_burners', sa.Boolean(), nullable=True, comment='Indicates whether the unit has duct-burners for supplementary firing of the turbine exhaust gas'), - sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), - sa.Column('topping_bottoming_code', sa.Text(), nullable=True, comment='If the generator is associated with a combined heat and power system, indicates whether the generator is part of a topping cycle or a bottoming cycle'), - sa.Column('solid_fuel_gasification', sa.Boolean(), nullable=True, comment='Indicates whether the generator is part of a solid fuel gasification system'), - sa.Column('pulverized_coal_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses pulverized coal technology'), - sa.Column('fluidized_bed_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses fluidized bed technology'), - sa.Column('subcritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses subcritical technology'), - sa.Column('supercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses supercritical technology'), - sa.Column('ultrasupercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses ultra-supercritical technology'), - sa.Column('stoker_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses stoker technology'), - sa.Column('other_combustion_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses other combustion technologies'), - sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), - sa.Column('rto_iso_lmp_node_id', sa.Text(), nullable=True, comment='The designation used to identify the price node in RTO/ISO Locational Marginal Price reports'), - sa.Column('rto_iso_location_wholesale_reporting_id', sa.Text(), nullable=True, comment='The designation used to report ths specific location of the wholesale sales transactions to FERC for the Electric Quarterly Report'), - sa.Column('associated_combined_heat_power', sa.Boolean(), nullable=True, comment='Indicates whether the generator is associated with a combined heat and power system'), - sa.Column('original_planned_generator_operating_date', sa.Date(), nullable=True, comment='The date the generator was originally scheduled to be operational'), - sa.Column('operating_switch', sa.Text(), nullable=True, comment='Indicates whether the fuel switching generator can switch when operating'), - sa.Column('previously_canceled', sa.Boolean(), nullable=True, comment='Indicates whether the generator was previously reported as indefinitely postponed or canceled'), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_generators_entity_eia_plant_id_eia_plants_entity_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', name=op.f('pk_generators_entity_eia')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_energy_efficiency_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('green_pricing_eia861', + op.create_table('core_eia861__yearly_green_pricing', sa.Column('customer_class', sa.Enum('commercial', 'industrial', 'direct_connection', 'other', 'residential', 'total', 'transportation'), nullable=True, comment='High level categorization of customer type.'), sa.Column('customers', sa.Float(), nullable=True, comment='Number of customers.'), sa.Column('green_pricing_revenue', sa.Float(), nullable=True), @@ -828,19 +596,9 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_green_pricing_eia861_data_maturity_data_maturities')) - ) - op.create_table('heat_rate_by_unit_monthly', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=False, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_heat_rate_by_unit_monthly_plant_id_eia_plants_entity_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'unit_id_pudl', name=op.f('pk_heat_rate_by_unit_monthly')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_green_pricing_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('mergers_eia861', + op.create_table('core_eia861__yearly_mergers', sa.Column('entity_type', sa.Text(), nullable=True, comment='Entity type of principal owner.'), sa.Column('merge_address', sa.Text(), nullable=True), sa.Column('merge_city', sa.Text(), nullable=True), @@ -855,9 +613,9 @@ def upgrade() -> None: sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), sa.Column('zip_code_4', sa.Text(), nullable=True, comment='Four digit US Zip Code suffix.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_mergers_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_mergers_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('net_metering_customer_fuel_class_eia861', + op.create_table('core_eia861__yearly_net_metering_customer_fuel_class', sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), sa.Column('customer_class', sa.Enum('commercial', 'industrial', 'direct_connection', 'other', 'residential', 'total', 'transportation'), nullable=True, comment='High level categorization of customer type.'), @@ -871,18 +629,18 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_net_metering_customer_fuel_class_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_net_metering_customer_fuel_class_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('net_metering_misc_eia861', + op.create_table('core_eia861__yearly_net_metering_misc', sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), sa.Column('pv_current_flow_type', sa.Enum('AC', 'DC'), nullable=True), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_net_metering_misc_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_net_metering_misc_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('non_net_metering_customer_fuel_class_eia861', + op.create_table('core_eia861__yearly_non_net_metering_customer_fuel_class', sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), sa.Column('customer_class', sa.Enum('commercial', 'industrial', 'direct_connection', 'other', 'residential', 'total', 'transportation'), nullable=True, comment='High level categorization of customer type.'), @@ -892,9 +650,9 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_non_net_metering_customer_fuel_class_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_non_net_metering_customer_fuel_class_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('non_net_metering_misc_eia861', + op.create_table('core_eia861__yearly_non_net_metering_misc', sa.Column('backup_capacity_mw', sa.Float(), nullable=True), sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), sa.Column('generators_number', sa.Float(), nullable=True), @@ -904,9 +662,9 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_owned_capacity_mw', sa.Float(), nullable=True), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_non_net_metering_misc_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_non_net_metering_misc_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('operational_data_misc_eia861', + op.create_table('core_eia861__yearly_operational_data_misc', sa.Column('consumed_by_facility_mwh', sa.Float(), nullable=True), sa.Column('consumed_by_respondent_without_charge_mwh', sa.Float(), nullable=True), sa.Column('data_observed', sa.Boolean(), nullable=True, comment='Is the value observed (True) or imputed (False).'), @@ -935,9 +693,9 @@ def upgrade() -> None: sa.Column('wholesale_power_purchases_mwh', sa.Float(), nullable=True), sa.Column('winter_peak_demand_mw', sa.Float(), nullable=True), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_operational_data_misc_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_operational_data_misc_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('operational_data_revenue_eia861', + op.create_table('core_eia861__yearly_operational_data_revenue', sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), sa.Column('revenue', sa.Float(), nullable=True), @@ -945,16 +703,9 @@ def upgrade() -> None: sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_operational_data_revenue_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_operational_data_revenue_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('plants_eia', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_plants_eia_plant_id_pudl_plants_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', name=op.f('pk_plants_eia')) - ) - op.create_table('reliability_eia861', + op.create_table('core_eia861__yearly_reliability', sa.Column('caidi_w_major_event_days_minus_loss_of_service_minutes', sa.Float(), nullable=True), sa.Column('caidi_w_major_event_days_minutes', sa.Float(), nullable=True), sa.Column('caidi_wo_major_event_days_minutes', sa.Float(), nullable=True), @@ -977,10 +728,10 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_reliability_eia861_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['momentary_interruption_definition'], ['momentary_interruptions_eia.code'], name=op.f('fk_reliability_eia861_momentary_interruption_definition_momentary_interruptions_eia')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_reliability_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['momentary_interruption_definition'], ['core_eia__codes_momentary_interruptions.code'], name=op.f('fk_core_eia861__yearly_reliability_momentary_interruption_definition_core_eia__codes_momentary_interruptions')) ) - op.create_table('sales_eia861', + op.create_table('core_eia861__yearly_sales', sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('state', sa.Text(), nullable=False, comment='Two letter US state abbreviation.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), @@ -996,10 +747,10 @@ def upgrade() -> None: sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), sa.Column('sales_revenue', sa.Float(), nullable=True, comment='Revenue from electricity sold.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_sales_eia861_data_maturity_data_maturities')), - sa.PrimaryKeyConstraint('utility_id_eia', 'state', 'report_date', 'balancing_authority_code_eia', 'customer_class', 'business_model', 'service_type', name=op.f('pk_sales_eia861')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_sales_data_maturity_core_pudl__codes_data_maturities')), + sa.PrimaryKeyConstraint('utility_id_eia', 'state', 'report_date', 'balancing_authority_code_eia', 'customer_class', 'business_model', 'service_type', name=op.f('pk_core_eia861__yearly_sales')) ) - op.create_table('service_territory_eia861', + op.create_table('core_eia861__yearly_service_territory', sa.Column('county', sa.Text(), nullable=True, comment='County name.'), sa.Column('short_form', sa.Boolean(), nullable=True), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), @@ -1009,74 +760,10 @@ def upgrade() -> None: sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), sa.Column('county_id_fips', sa.Text(), nullable=False, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_service_territory_eia861_data_maturity_data_maturities')), - sa.PrimaryKeyConstraint('report_date', 'utility_id_eia', 'county_id_fips', name=op.f('pk_service_territory_eia861')) - ) - op.create_table('summarized_demand_ferc714', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False), - sa.Column('demand_annual_mwh', sa.Float(), nullable=True), - sa.Column('population', sa.Float(), nullable=True, comment='County population, sourced from Census DP1 data.'), - sa.Column('area_km2', sa.Float(), nullable=True, comment='County area in km2.'), - sa.Column('population_density_km2', sa.Float(), nullable=True, comment='Average population per sq. km area of a service territory.'), - sa.Column('demand_annual_per_capita_mwh', sa.Float(), nullable=True, comment='Per-capita annual demand, averaged using Census county-level population estimates.'), - sa.Column('demand_density_mwh_km2', sa.Float(), nullable=True, comment='Annual demand per km2 of a given service territory.'), - sa.Column('eia_code', sa.Integer(), nullable=True), - sa.Column('respondent_type', sa.Enum('utility', 'balancing_authority'), nullable=True), - sa.Column('respondent_name_ferc714', sa.Text(), nullable=True), - sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=True, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), - sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), - sa.Column('balancing_authority_name_eia', sa.Text(), nullable=True, comment='Name of the balancing authority.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['respondent_id_ferc714.respondent_id_ferc714'], name=op.f('fk_summarized_demand_ferc714_respondent_id_ferc714_respondent_id_ferc714')), - sa.PrimaryKeyConstraint('respondent_id_ferc714', 'report_date', name=op.f('pk_summarized_demand_ferc714')) - ) - op.create_table('utilities_eia', - sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_utilities_eia_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_eia', name=op.f('pk_utilities_eia')) - ) - op.create_table('utilities_eia860', - sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('street_address', sa.Text(), nullable=True), - sa.Column('city', sa.Text(), nullable=True), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), - sa.Column('plants_reported_owner', sa.Boolean(), nullable=True, comment='Is the reporting entity an owner of power plants reported on Schedule 2 of the form?'), - sa.Column('plants_reported_operator', sa.Boolean(), nullable=True, comment='Is the reporting entity an operator of power plants reported on Schedule 2 of the form?'), - sa.Column('plants_reported_asset_manager', sa.Boolean(), nullable=True, comment='Is the reporting entity an asset manager of power plants reported on Schedule 2 of the form?'), - sa.Column('plants_reported_other_relationship', sa.Boolean(), nullable=True, comment='Does the reporting entity have any other relationship to the power plants reported on Schedule 2 of the form?'), - sa.Column('entity_type', sa.Text(), nullable=True, comment='Entity type of principal owner.'), - sa.Column('attention_line', sa.Text(), nullable=True), - sa.Column('address_2', sa.Text(), nullable=True), - sa.Column('zip_code_4', sa.Text(), nullable=True, comment='Four digit US Zip Code suffix.'), - sa.Column('contact_firstname', sa.Text(), nullable=True, comment='First name of utility contact 1.'), - sa.Column('contact_lastname', sa.Text(), nullable=True, comment='Last name of utility contact 1.'), - sa.Column('contact_title', sa.Text(), nullable=True, comment='Title of of utility contact 1.'), - sa.Column('phone_number', sa.Text(), nullable=True, comment='Phone number for utility contact 1.'), - sa.Column('phone_extension', sa.Text(), nullable=True, comment='Phone extension for utility contact 1'), - sa.Column('contact_firstname_2', sa.Text(), nullable=True, comment='First name of utility contact 2.'), - sa.Column('contact_lastname_2', sa.Text(), nullable=True, comment='Last name of utility contact 2.'), - sa.Column('contact_title_2', sa.Text(), nullable=True, comment='Title of utility contact 2.'), - sa.Column('phone_number_2', sa.Text(), nullable=True, comment='Phone number for utility contact 2.'), - sa.Column('phone_extension_2', sa.Text(), nullable=True, comment='Phone extension for utility contact 2'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_utilities_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_utilities_eia860_utility_id_eia_utilities_entity_eia')), - sa.PrimaryKeyConstraint('utility_id_eia', 'report_date', name=op.f('pk_utilities_eia860')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_service_territory_data_maturity_core_pudl__codes_data_maturities')), + sa.PrimaryKeyConstraint('report_date', 'utility_id_eia', 'county_id_fips', name=op.f('pk_core_eia861__yearly_service_territory')) ) - op.create_table('utilities_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_utilities_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', name=op.f('pk_utilities_ferc1')) - ) - op.create_table('utility_data_misc_eia861', + op.create_table('core_eia861__yearly_utility_data_misc', sa.Column('alternative_fuel_vehicle_2_activity', sa.Boolean(), nullable=True), sa.Column('alternative_fuel_vehicle_activity', sa.Boolean(), nullable=True), sa.Column('bundled_activity', sa.Boolean(), nullable=True), @@ -1096,129 +783,230 @@ def upgrade() -> None: sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('wholesale_marketing_activity', sa.Boolean(), nullable=True), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_utility_data_misc_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_utility_data_misc_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('utility_data_nerc_eia861', + op.create_table('core_eia861__yearly_utility_data_nerc', sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'), sa.Column('nerc_regions_of_operation', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_utility_data_nerc_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_utility_data_nerc_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('utility_data_rto_eia861', + op.create_table('core_eia861__yearly_utility_data_rto', sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), sa.Column('rtos_of_operation', sa.Enum('caiso', 'ercot', 'isone', 'miso', 'nyiso', 'other', 'pjm', 'spp'), nullable=True), sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_utility_data_rto_eia861_data_maturity_data_maturities')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia861__yearly_utility_data_rto_data_maturity_core_pudl__codes_data_maturities')) ) - op.create_table('utility_plant_assn', - sa.Column('utility_id_pudl', sa.Integer(), nullable=False, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=False, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_utility_plant_assn_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_utility_plant_assn_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_pudl', 'plant_id_pudl', name=op.f('pk_utility_plant_assn')) - ) - op.create_table('balance_sheet_assets_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('asset_type', sa.Text(), nullable=False, comment='Type of asset being reported to the balance_sheet_assets_ferc1 table.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_balance_sheet_assets_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'asset_type', name=op.f('pk_balance_sheet_assets_ferc1')) - ) - op.create_table('balance_sheet_liabilities_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('liability_type', sa.Text(), nullable=False, comment='Type of liability being reported to the balance_sheet_liabilities_ferc1 table.'), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_balance_sheet_liabilities_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'liability_type', name=op.f('pk_balance_sheet_liabilities_ferc1')) + op.create_table('core_eia923__entity_coalmine', + sa.Column('mine_id_pudl', sa.Integer(), nullable=False, comment='Dynamically assigned PUDL mine identifier.'), + sa.Column('mine_name', sa.Text(), nullable=True, comment='Coal mine name.'), + sa.Column('mine_type_code', sa.Text(), nullable=True, comment='Type of coal mine.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('county_id_fips', sa.Text(), nullable=True, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), + sa.Column('mine_id_msha', sa.Integer(), nullable=True, comment='MSHA issued mine identifier.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia923__entity_coalmine_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['mine_type_code'], ['core_eia__codes_coalmine_types.code'], name=op.f('fk_core_eia923__entity_coalmine_mine_type_code_core_eia__codes_coalmine_types')), + sa.PrimaryKeyConstraint('mine_id_pudl', name=op.f('pk_core_eia923__entity_coalmine')) ) - op.create_table('boiler_fuel_eia923', + op.create_table('core_eia923__monthly_generation_fuel', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_type_code_aer', sa.Text(), nullable=True, comment='A partial aggregation of the reported fuel type codes into larger categories used by EIA in, for example, the Annual Energy Review (AER). Two or three letter alphanumeric.'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['fuel_type_code_aer'], ['core_eia__codes_fuel_types_aer.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_fuel_type_code_aer_core_eia__codes_fuel_types_aer')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia923__monthly_generation_fuel_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_core_eia923__monthly_generation_fuel')) + ) + op.create_table('core_eia923__monthly_generation_fuel_nuclear', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('nuclear_unit_id', sa.Text(), nullable=False, comment='For nuclear plants only, the unit number .One digit numeric. Nuclear plants are the only type of plants for which data are shown explicitly at the generating unit level.'), + sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_type_code_aer', sa.Text(), nullable=True, comment='A partial aggregation of the reported fuel type codes into larger categories used by EIA in, for example, the Annual Energy Review (AER). Two or three letter alphanumeric.'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), - sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_boiler_fuel_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['boilers_entity_eia.plant_id_eia', 'boilers_entity_eia.boiler_id'], name=op.f('fk_boiler_fuel_eia923_plant_id_eia_boilers_entity_eia')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_boiler_fuel_eia923_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_boiler_fuel_eia923')) + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_nuclear_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_nuclear_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['fuel_type_code_aer'], ['core_eia__codes_fuel_types_aer.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_nuclear_fuel_type_code_aer_core_eia__codes_fuel_types_aer')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia923__monthly_generation_fuel_nuclear_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_core_eia923__monthly_generation_fuel_nuclear_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'nuclear_unit_id', 'energy_source_code', 'prime_mover_code', name=op.f('pk_core_eia923__monthly_generation_fuel_nuclear')) + ) + op.create_table('core_eia__entity_boilers', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), + sa.Column('boiler_manufacturer', sa.Text(), nullable=True, comment='Name of boiler manufacturer.'), + sa.Column('boiler_manufacturer_code', sa.Text(), nullable=True, comment='EIA short code for boiler manufacturer.'), + sa.ForeignKeyConstraint(['boiler_manufacturer_code'], ['core_eia__codes_environmental_equipment_manufacturers.code'], name=op.f('fk_core_eia__entity_boilers_boiler_manufacturer_code_core_eia__codes_environmental_equipment_manufacturers')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia__entity_boilers_plant_id_eia_core_eia__entity_plants')), + sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', name=op.f('pk_core_eia__entity_boilers')) ) - op.create_table('capacity_factor_by_generator_monthly', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + op.create_table('core_eia__entity_generators', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_capacity_factor_by_generator_monthly_plant_id_eia_generators_entity_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_capacity_factor_by_generator_monthly')) + sa.Column('duct_burners', sa.Boolean(), nullable=True, comment='Indicates whether the unit has duct-burners for supplementary firing of the turbine exhaust gas'), + sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), + sa.Column('topping_bottoming_code', sa.Text(), nullable=True, comment='If the generator is associated with a combined heat and power system, indicates whether the generator is part of a topping cycle or a bottoming cycle'), + sa.Column('solid_fuel_gasification', sa.Boolean(), nullable=True, comment='Indicates whether the generator is part of a solid fuel gasification system'), + sa.Column('pulverized_coal_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses pulverized coal technology'), + sa.Column('fluidized_bed_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses fluidized bed technology'), + sa.Column('subcritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses subcritical technology'), + sa.Column('supercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses supercritical technology'), + sa.Column('ultrasupercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses ultra-supercritical technology'), + sa.Column('stoker_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses stoker technology'), + sa.Column('other_combustion_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses other combustion technologies'), + sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), + sa.Column('rto_iso_lmp_node_id', sa.Text(), nullable=True, comment='The designation used to identify the price node in RTO/ISO Locational Marginal Price reports'), + sa.Column('rto_iso_location_wholesale_reporting_id', sa.Text(), nullable=True, comment='The designation used to report ths specific location of the wholesale sales transactions to FERC for the Electric Quarterly Report'), + sa.Column('associated_combined_heat_power', sa.Boolean(), nullable=True, comment='Indicates whether the generator is associated with a combined heat and power system'), + sa.Column('original_planned_generator_operating_date', sa.Date(), nullable=True, comment='The date the generator was originally scheduled to be operational'), + sa.Column('operating_switch', sa.Text(), nullable=True, comment='Indicates whether the fuel switching generator can switch when operating'), + sa.Column('previously_canceled', sa.Boolean(), nullable=True, comment='Indicates whether the generator was previously reported as indefinitely postponed or canceled'), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia__entity_generators_plant_id_eia_core_eia__entity_plants')), + sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', name=op.f('pk_core_eia__entity_generators')) ) - op.create_table('cash_flow_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('amount_type', sa.Text(), nullable=False, comment='Label describing the type of amount being reported. This could be a balance or a change in value.'), - sa.Column('amount', sa.Float(), nullable=True, comment='Reported amount of dollars. This could be a balance or a change in value.'), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_cash_flow_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'amount_type', name=op.f('pk_cash_flow_ferc1')) + op.create_table('core_ferc714__hourly_demand_pa', + sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False, comment='FERC Form 714 respondent ID. Note that this ID does not correspond to FERC respondent IDs from other forms.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('utc_datetime', sqlite.DATETIME(), nullable=False), + sa.Column('timezone', sa.Enum('America/New_York', 'America/Chicago', 'America/Denver', 'America/Los_Angeles', 'America/Anchorage', 'Pacific/Honolulu'), nullable=True, comment='IANA timezone name'), + sa.Column('demand_mwh', sa.Float(), nullable=True), + sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['core_ferc714__respondent_id.respondent_id_ferc714'], name=op.f('fk_core_ferc714__hourly_demand_pa_respondent_id_ferc714_core_ferc714__respondent_id')), + sa.PrimaryKeyConstraint('respondent_id_ferc714', 'utc_datetime', name=op.f('pk_core_ferc714__hourly_demand_pa')) ) - op.create_table('denorm_balance_sheet_assets_ferc1', - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + op.create_table('core_pudl__assn_eia_pudl_plants', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_core_pudl__assn_eia_pudl_plants_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', name=op.f('pk_core_pudl__assn_eia_pudl_plants')) + ) + op.create_table('core_pudl__assn_eia_pudl_utilities', + sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('asset_type', sa.Text(), nullable=False, comment='Type of asset being reported to the balance_sheet_assets_ferc1 table.'), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_balance_sheet_assets_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_balance_sheet_assets_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'asset_type', name=op.f('pk_denorm_balance_sheet_assets_ferc1')) + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_core_pudl__assn_eia_pudl_utilities_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_eia', name=op.f('pk_core_pudl__assn_eia_pudl_utilities')) ) - op.create_table('denorm_balance_sheet_liabilities_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + op.create_table('core_pudl__assn_ferc1_pudl_utilities', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('liability_type', sa.Text(), nullable=False, comment='Type of liability being reported to the balance_sheet_liabilities_ferc1 table.'), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_balance_sheet_liabilities_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_balance_sheet_liabilities_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'liability_type', name=op.f('pk_denorm_balance_sheet_liabilities_ferc1')) + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_core_pudl__assn_ferc1_pudl_utilities_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', name=op.f('pk_core_pudl__assn_ferc1_pudl_utilities')) + ) + op.create_table('core_pudl__assn_utilities_plants', + sa.Column('utility_id_pudl', sa.Integer(), nullable=False, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=False, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_core_pudl__assn_utilities_plants_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_core_pudl__assn_utilities_plants_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_pudl', 'plant_id_pudl', name=op.f('pk_core_pudl__assn_utilities_plants')) ) - op.create_table('denorm_boiler_fuel_eia923', + op.create_table('out_eia860__yearly_emissions_control_equipment', + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('emission_control_id_pudl', sa.Float(), nullable=False, comment='A PUDL-generated ID used to distinguish emission control units in the same report year and plant id. This ID should not be used to track units over time or between plants.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.Column('emission_control_equipment_type_code', sa.Text(), nullable=True, comment='Short code indicating the type of emission control equipment installed.'), + sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), + sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), + sa.Column('mercury_control_id_eia', sa.Text(), nullable=True, comment='Mercury control identification number. This ID is not a unique identifier.'), + sa.Column('nox_control_id_eia', sa.Text(), nullable=True, comment='Nitrogen oxide control identification number. This ID is not a unique identifier.'), + sa.Column('particulate_control_id_eia', sa.Text(), nullable=True, comment='Particulate matter control identification number. This ID is not a unique identifier.'), + sa.Column('so2_control_id_eia', sa.Text(), nullable=True, comment='Sulfur dioxide control identification number. This ID is not a unique identifier.'), + sa.Column('acid_gas_control', sa.Boolean(), nullable=True, comment='Indicates whether the emissions control equipment controls acid (HCl) gas.'), + sa.Column('emission_control_equipment_cost', sa.Float(), nullable=True, comment='The total cost to install a piece of emission control equipment.'), + sa.Column('emission_control_operating_date', sa.Date(), nullable=True, comment='The date a piece of emissions control equipment began operating. Derived from month and year columns in the raw data.'), + sa.Column('emission_control_retirement_date', sa.Date(), nullable=True, comment='The expected or actual retirement date for a piece of emissions control equipment. Derived from month and year columns in the raw data.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia860__yearly_emissions_control_equipment_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['emission_control_equipment_type_code'], ['core_eia__codes_emission_control_equipment_types.code'], name=op.f('fk_out_eia860__yearly_emissions_control_equipment_emission_control_equipment_type_code_core_eia__codes_emission_control_equipment_types')), + sa.ForeignKeyConstraint(['operational_status_code'], ['core_eia__codes_operational_status.code'], name=op.f('fk_out_eia860__yearly_emissions_control_equipment_operational_status_code_core_eia__codes_operational_status')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_out_eia860__yearly_emissions_control_equipment_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia860__yearly_emissions_control_equipment_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia860__yearly_emissions_control_equipment_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia860__yearly_emissions_control_equipment_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_year', 'plant_id_eia', 'emission_control_id_pudl', name=op.f('pk_out_eia860__yearly_emissions_control_equipment')) + ) + op.create_table('out_eia923__fuel_receipts_costs', + sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('contract_type_code', sa.Enum('S', 'C', 'NC', 'T'), nullable=True, comment='Purchase type under which receipts occurred in the reporting month. C: Contract, NC: New Contract, S: Spot Purchase, T: Tolling Agreement.'), + sa.Column('contract_expiration_date', sa.Date(), nullable=True, comment='Date contract expires.Format: MMYY.'), + sa.Column('energy_source_code', sa.Text(), nullable=True, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_group_code', sa.Enum('petroleum', 'other_gas', 'petroleum_coke', 'natural_gas', 'coal'), nullable=True, comment='Fuel groups used in the Electric Power Monthly'), + sa.Column('supplier_name', sa.Text(), nullable=True, comment='Company that sold the fuel to the plant or, in the case of Natural Gas, pipline owner.'), + sa.Column('fuel_received_units', sa.Float(), nullable=True, comment='Quanity of fuel received in tons, barrel, or Mcf.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('bulk_agg_fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Fuel cost per mmbtu reported in the EIA bulk electricity data. This is an aggregate average fuel price for a whole state, region, month, sector, etc. Used to fill in missing fuel prices.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), + sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), + sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), + sa.Column('mercury_content_ppm', sa.Float(), nullable=True, comment='Mercury content in parts per million (ppm) to the nearest 0.001 ppm.'), + sa.Column('primary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the longest distance transported.'), + sa.Column('secondary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the second longest distance transported.'), + sa.Column('natural_gas_transport_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natural gas transportation service.'), + sa.Column('natural_gas_delivery_contract_type_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natrual gas delivery service:'), + sa.Column('moisture_content_pct', sa.Float(), nullable=True), + sa.Column('chlorine_content_ppm', sa.Float(), nullable=True), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.Column('mine_id_msha', sa.Integer(), nullable=True, comment='MSHA issued mine identifier.'), + sa.Column('mine_name', sa.Text(), nullable=True, comment='Coal mine name.'), + sa.Column('mine_state', sa.Text(), nullable=True, comment='State where the coal mine is located. Two letter abbreviation.'), + sa.Column('coalmine_county_id_fips', sa.Text(), nullable=True, comment='County ID from the Federal Information Processing Standard Publication 6-4. This is the county where the coal mine is located.'), + sa.Column('mine_type_code', sa.Text(), nullable=True, comment='Type of coal mine.'), + sa.ForeignKeyConstraint(['contract_type_code'], ['core_eia__codes_contract_types.code'], name=op.f('fk_out_eia923__fuel_receipts_costs_contract_type_code_core_eia__codes_contract_types')), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__fuel_receipts_costs_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__fuel_receipts_costs_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['mine_type_code'], ['core_eia__codes_coalmine_types.code'], name=op.f('fk_out_eia923__fuel_receipts_costs_mine_type_code_core_eia__codes_coalmine_types')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_out_eia923__fuel_receipts_costs_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__fuel_receipts_costs_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['primary_transportation_mode_code'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia923__fuel_receipts_costs_primary_transportation_mode_code_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['secondary_transportation_mode_code'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia923__fuel_receipts_costs_secondary_transportation_mode_code_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__fuel_receipts_costs_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__fuel_receipts_costs_utility_id_pudl_core_pudl__entity_utilities_pudl')) + ) + op.create_table('out_eia923__generation_fuel_combined', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), @@ -1226,25 +1014,56 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_type_code_aer', sa.Text(), nullable=True, comment='A partial aggregation of the reported fuel type codes into larger categories used by EIA in, for example, the Annual Energy Review (AER). Two or three letter alphanumeric.'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__generation_fuel_combined_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__generation_fuel_combined_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['fuel_type_code_aer'], ['core_eia__codes_fuel_types_aer.code'], name=op.f('fk_out_eia923__generation_fuel_combined_fuel_type_code_aer_core_eia__codes_fuel_types_aer')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_out_eia923__generation_fuel_combined_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__generation_fuel_combined_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__generation_fuel_combined_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__generation_fuel_combined_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__generation_fuel_combined_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_out_eia923__generation_fuel_combined')) + ) + op.create_table('out_eia923__monthly_fuel_receipts_costs', + sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_received_units', sa.Float(), nullable=True, comment='Quanity of fuel received in tons, barrel, or Mcf.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_denorm_boiler_fuel_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['boilers_entity_eia.plant_id_eia', 'boilers_entity_eia.boiler_id'], name=op.f('fk_denorm_boiler_fuel_eia923_plant_id_eia_boilers_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_boiler_fuel_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_boiler_fuel_eia923_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_boiler_fuel_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_boiler_fuel_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_denorm_boiler_fuel_eia923')) - ) - op.create_table('denorm_boiler_fuel_monthly_eia923', + sa.Column('mercury_content_ppm', sa.Float(), nullable=True, comment='Mercury content in parts per million (ppm) to the nearest 0.001 ppm.'), + sa.Column('moisture_content_pct', sa.Float(), nullable=True), + sa.Column('chlorine_content_ppm', sa.Float(), nullable=True), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__monthly_fuel_receipts_costs_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_out_eia923__monthly_fuel_receipts_costs_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__monthly_fuel_receipts_costs_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__monthly_fuel_receipts_costs_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__monthly_fuel_receipts_costs_utility_id_pudl_core_pudl__entity_utilities_pudl')) + ) + op.create_table('out_eia923__monthly_generation_fuel_combined', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), @@ -1252,218 +1071,384 @@ def upgrade() -> None: sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), + sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_units', sa.Float(), nullable=True, comment='Consumption for electric generation of the fuel type in physical unit.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__monthly_generation_fuel_combined_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_out_eia923__monthly_generation_fuel_combined')) + ) + op.create_table('out_ferc714__respondents_with_fips', + sa.Column('eia_code', sa.Integer(), nullable=True, comment='EIA utility or balancing area authority ID associated with this FERC Form 714 respondent. Note that many utilities are also balancing authorities and in many cases EIA uses the same integer ID to identify a utility in its role as a balancing authority AND as a utility, but there is no requirement that these IDs be the same, and in a number of cases they are different.'), + sa.Column('respondent_type', sa.Enum('utility', 'balancing_authority'), nullable=True), + sa.Column('respondent_id_ferc714', sa.Integer(), nullable=True, comment='FERC Form 714 respondent ID. Note that this ID does not correspond to FERC respondent IDs from other forms.'), + sa.Column('respondent_name_ferc714', sa.Text(), nullable=True, comment='Name of the utility, balancing area authority, or planning authority responding to FERC Form 714.'), + sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), + sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=True, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), + sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), + sa.Column('balancing_authority_name_eia', sa.Text(), nullable=True, comment='Name of the balancing authority.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('county', sa.Text(), nullable=True, comment='County name.'), + sa.Column('state_id_fips', sa.Text(), nullable=True, comment='Two digit state FIPS code.'), + sa.Column('county_id_fips', sa.Text(), nullable=True, comment='County ID from the Federal Information Processing Standard Publication 6-4.'), + sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['core_ferc714__respondent_id.respondent_id_ferc714'], name=op.f('fk_out_ferc714__respondents_with_fips_respondent_id_ferc714_core_ferc714__respondent_id')) + ) + op.create_table('out_ferc714__summarized_demand', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('respondent_id_ferc714', sa.Integer(), nullable=False, comment='FERC Form 714 respondent ID. Note that this ID does not correspond to FERC respondent IDs from other forms.'), + sa.Column('demand_annual_mwh', sa.Float(), nullable=True), + sa.Column('population', sa.Float(), nullable=True, comment='County population, sourced from Census DP1 data.'), + sa.Column('area_km2', sa.Float(), nullable=True, comment='County area in km2.'), + sa.Column('population_density_km2', sa.Float(), nullable=True, comment='Average population per sq. km area of a service territory.'), + sa.Column('demand_annual_per_capita_mwh', sa.Float(), nullable=True, comment='Per-capita annual demand, averaged using Census county-level population estimates.'), + sa.Column('demand_density_mwh_km2', sa.Float(), nullable=True, comment='Annual demand per km2 of a given service territory.'), + sa.Column('eia_code', sa.Integer(), nullable=True, comment='EIA utility or balancing area authority ID associated with this FERC Form 714 respondent. Note that many utilities are also balancing authorities and in many cases EIA uses the same integer ID to identify a utility in its role as a balancing authority AND as a utility, but there is no requirement that these IDs be the same, and in a number of cases they are different.'), + sa.Column('respondent_type', sa.Enum('utility', 'balancing_authority'), nullable=True), + sa.Column('respondent_name_ferc714', sa.Text(), nullable=True, comment='Name of the utility, balancing area authority, or planning authority responding to FERC Form 714.'), + sa.Column('balancing_authority_id_eia', sa.Integer(), nullable=True, comment='EIA balancing authority ID. This is often (but not always!) the same as the utility ID associated with the same legal entity.'), + sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), + sa.Column('balancing_authority_name_eia', sa.Text(), nullable=True, comment='Name of the balancing authority.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.ForeignKeyConstraint(['respondent_id_ferc714'], ['core_ferc714__respondent_id.respondent_id_ferc714'], name=op.f('fk_out_ferc714__summarized_demand_respondent_id_ferc714_core_ferc714__respondent_id')), + sa.PrimaryKeyConstraint('respondent_id_ferc714', 'report_date', name=op.f('pk_out_ferc714__summarized_demand')) + ) + op.create_table('_out_eia__monthly_capacity_factor_by_generator', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk__out_eia__monthly_capacity_factor_by_generator_plant_id_eia_core_eia__entity_generators')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__monthly_capacity_factor_by_generator')) + ) + op.create_table('_out_eia__monthly_derived_generator_attributes', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk__out_eia__monthly_derived_generator_attributes_plant_id_eia_core_eia__entity_generators')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__monthly_derived_generator_attributes')) + ) + op.create_table('_out_eia__monthly_fuel_cost_by_generator', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk__out_eia__monthly_fuel_cost_by_generator_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_eia__monthly_fuel_cost_by_generator_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk__out_eia__monthly_fuel_cost_by_generator_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_eia__monthly_fuel_cost_by_generator_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__monthly_fuel_cost_by_generator')) + ) + op.create_table('_out_eia__monthly_heat_rate_by_generator', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), + sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk__out_eia__monthly_heat_rate_by_generator_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk__out_eia__monthly_heat_rate_by_generator_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__monthly_heat_rate_by_generator')) + ) + op.create_table('core_eia860__scd_plants', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('ash_impoundment', sa.Boolean(), nullable=True, comment='Is there an ash impoundment (e.g. pond, reservoir) at the plant?'), + sa.Column('ash_impoundment_lined', sa.Boolean(), nullable=True, comment='If there is an ash impoundment at the plant, is the impoundment lined?'), + sa.Column('ash_impoundment_status', sa.Text(), nullable=True, comment='If there is an ash impoundment at the plant, the ash impoundment status as of December 31 of the reporting year.'), + sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), + sa.Column('balancing_authority_name_eia', sa.Text(), nullable=True, comment='Name of the balancing authority.'), + sa.Column('datum', sa.Text(), nullable=True, comment='Geodetic coordinate system identifier (e.g. NAD27, NAD83, or WGS84).'), + sa.Column('energy_storage', sa.Boolean(), nullable=True, comment='Indicates if the facility has energy storage capabilities.'), + sa.Column('ferc_cogen_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC cogenerator status. See FERC Form 556.'), + sa.Column('ferc_cogen_status', sa.Boolean(), nullable=True, comment='Indicates whether the plant has FERC qualifying facility cogenerator status. See FERC Form 556.'), + sa.Column('ferc_exempt_wholesale_generator_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC qualifying facility exempt wholesale generator status.'), + sa.Column('ferc_exempt_wholesale_generator', sa.Boolean(), nullable=True, comment='Indicates whether the plant has FERC qualifying facility exempt wholesale generator status'), + sa.Column('ferc_small_power_producer_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC qualifying facility small power producer status. See FERC Form 556.'), + sa.Column('ferc_small_power_producer', sa.Boolean(), nullable=True, comment='Indicates whether the plant has FERC qualifying facility small power producer status. See FERC Form 556.'), + sa.Column('ferc_qualifying_facility_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC qualifying facility cogenerator status. See FERC Form 556.'), + sa.Column('grid_voltage_1_kv', sa.Float(), nullable=True, comment="Plant's grid voltage at point of interconnection to transmission or distibution facilities"), + sa.Column('grid_voltage_2_kv', sa.Float(), nullable=True, comment="Plant's grid voltage at point of interconnection to transmission or distibution facilities"), + sa.Column('grid_voltage_3_kv', sa.Float(), nullable=True, comment="Plant's grid voltage at point of interconnection to transmission or distibution facilities"), + sa.Column('iso_rto_code', sa.Text(), nullable=True, comment="The code of the plant's ISO or RTO. NA if not reported in that year."), + sa.Column('liquefied_natural_gas_storage', sa.Boolean(), nullable=True, comment='Indicates if the facility have the capability to store the natural gas in the form of liquefied natural gas.'), + sa.Column('natural_gas_local_distribution_company', sa.Text(), nullable=True, comment='Names of Local Distribution Company (LDC), connected to natural gas burning power plants.'), + sa.Column('natural_gas_storage', sa.Boolean(), nullable=True, comment='Indicates if the facility have on-site storage of natural gas.'), + sa.Column('natural_gas_pipeline_name_1', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'), + sa.Column('natural_gas_pipeline_name_2', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'), + sa.Column('natural_gas_pipeline_name_3', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'), + sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'), + sa.Column('net_metering', sa.Boolean(), nullable=True, comment='Did this plant have a net metering agreement in effect during the reporting year? (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'), + sa.Column('pipeline_notes', sa.Text(), nullable=True, comment='Additional owner or operator of natural gas pipeline.'), + sa.Column('primary_purpose_id_naics', sa.Integer(), nullable=True, comment='North American Industry Classification System (NAICS) code that best describes the primary purpose of the reporting plant'), + sa.Column('regulatory_status_code', sa.Text(), nullable=True, comment='Indicates whether the plant is regulated or non-regulated.'), + sa.Column('reporting_frequency_code', sa.Enum('A', 'AM', 'M'), nullable=True, comment='Code that specifies what time period data has to be reported (i.e. monthly data or annual totals) and how often the power plant reports this data to EIA. See reporting_frequencies_eia for more details.'), + sa.Column('sector_id_eia', sa.Integer(), nullable=True, comment='EIA assigned sector ID, corresponding to high level NAICS sector, designated by the primary purpose, regulatory status and plant-level combined heat and power status'), + sa.Column('sector_name_eia', sa.Text(), nullable=True, comment='EIA assigned sector name, corresponding to high level NAICS sector, designated by the primary purpose, regulatory status and plant-level combined heat and power status'), + sa.Column('service_area', sa.Text(), nullable=True, comment="Service area in which plant is located; for unregulated companies, it's the electric utility with which plant is interconnected"), + sa.Column('transmission_distribution_owner_id', sa.Integer(), nullable=True, comment='EIA-assigned code for owner of transmission/distribution system to which the plant is interconnected.'), + sa.Column('transmission_distribution_owner_name', sa.Text(), nullable=True, comment='Name of the owner of the transmission or distribution system to which the plant is interconnected.'), + sa.Column('transmission_distribution_owner_state', sa.Text(), nullable=True, comment='State location for owner of transmission/distribution system to which the plant is interconnected.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('water_source', sa.Text(), nullable=True, comment='Name of water source associated with the plant.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['balancing_authority_code_eia'], ['core_eia__codes_balancing_authorities.code'], name=op.f('fk_core_eia860__scd_plants_balancing_authority_code_eia_core_eia__codes_balancing_authorities')), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__scd_plants_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia860__scd_plants_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['reporting_frequency_code'], ['core_eia__codes_reporting_frequencies.code'], name=op.f('fk_core_eia860__scd_plants_reporting_frequency_code_core_eia__codes_reporting_frequencies')), + sa.ForeignKeyConstraint(['sector_id_eia'], ['core_eia__codes_sector_consolidated.code'], name=op.f('fk_core_eia860__scd_plants_sector_id_eia_core_eia__codes_sector_consolidated')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_core_eia860__scd_plants_utility_id_eia_core_eia860__scd_utilities')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', name=op.f('pk_core_eia860__scd_plants')) + ) + op.create_table('core_eia923__monthly_boiler_fuel', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_denorm_boiler_fuel_monthly_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['boilers_entity_eia.plant_id_eia', 'boilers_entity_eia.boiler_id'], name=op.f('fk_denorm_boiler_fuel_monthly_eia923_plant_id_eia_boilers_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_boiler_fuel_monthly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_boiler_fuel_monthly_eia923_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_boiler_fuel_monthly_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_boiler_fuel_monthly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_denorm_boiler_fuel_monthly_eia923')) - ) - op.create_table('denorm_cash_flow_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia923__monthly_boiler_fuel_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia923__monthly_boiler_fuel_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['core_eia__entity_boilers.plant_id_eia', 'core_eia__entity_boilers.boiler_id'], name=op.f('fk_core_eia923__monthly_boiler_fuel_plant_id_eia_core_eia__entity_boilers')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_core_eia923__monthly_boiler_fuel_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_core_eia923__monthly_boiler_fuel')) + ) + op.create_table('core_eia923__monthly_fuel_receipts_costs', + sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), + sa.Column('contract_type_code', sa.Enum('S', 'C', 'NC', 'T'), nullable=True, comment='Purchase type under which receipts occurred in the reporting month. C: Contract, NC: New Contract, S: Spot Purchase, T: Tolling Agreement.'), + sa.Column('contract_expiration_date', sa.Date(), nullable=True, comment='Date contract expires.Format: MMYY.'), + sa.Column('energy_source_code', sa.Text(), nullable=True, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_group_code', sa.Enum('petroleum', 'other_gas', 'petroleum_coke', 'natural_gas', 'coal'), nullable=True, comment='Fuel groups used in the Electric Power Monthly'), + sa.Column('mine_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL mine identifier.'), + sa.Column('supplier_name', sa.Text(), nullable=True, comment='Company that sold the fuel to the plant or, in the case of Natural Gas, pipline owner.'), + sa.Column('fuel_received_units', sa.Float(), nullable=True, comment='Quanity of fuel received in tons, barrel, or Mcf.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), + sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), + sa.Column('mercury_content_ppm', sa.Float(), nullable=True, comment='Mercury content in parts per million (ppm) to the nearest 0.001 ppm.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('primary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the longest distance transported.'), + sa.Column('secondary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the second longest distance transported.'), + sa.Column('natural_gas_transport_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natural gas transportation service.'), + sa.Column('natural_gas_delivery_contract_type_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natrual gas delivery service:'), + sa.Column('moisture_content_pct', sa.Float(), nullable=True), + sa.Column('chlorine_content_ppm', sa.Float(), nullable=True), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['contract_type_code'], ['core_eia__codes_contract_types.code'], name=op.f('fk_core_eia923__monthly_fuel_receipts_costs_contract_type_code_core_eia__codes_contract_types')), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia923__monthly_fuel_receipts_costs_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia923__monthly_fuel_receipts_costs_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['mine_id_pudl'], ['core_eia923__entity_coalmine.mine_id_pudl'], name=op.f('fk_core_eia923__monthly_fuel_receipts_costs_mine_id_pudl_core_eia923__entity_coalmine')), + sa.ForeignKeyConstraint(['plant_id_eia'], ['core_eia__entity_plants.plant_id_eia'], name=op.f('fk_core_eia923__monthly_fuel_receipts_costs_plant_id_eia_core_eia__entity_plants')), + sa.ForeignKeyConstraint(['primary_transportation_mode_code'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia923__monthly_fuel_receipts_costs_primary_transportation_mode_code_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['secondary_transportation_mode_code'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia923__monthly_fuel_receipts_costs_secondary_transportation_mode_code_core_eia__codes_fuel_transportation_modes')) + ) + op.create_table('core_eia923__monthly_generation', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia923__monthly_generation_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_core_eia923__monthly_generation_plant_id_eia_core_eia__entity_generators')), + sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_core_eia923__monthly_generation')) + ) + op.create_table('core_epa__assn_eia_epacamd', + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('plant_id_epa', sa.Integer(), nullable=True, comment='The ORISPL ID used by EPA to refer to the plant. Usually but not always the same as plant_id_eia.'), + sa.Column('emissions_unit_id_epa', sa.Text(), nullable=True, comment='Emissions (smokestack) unit monitored by EPA CEMS.'), + sa.Column('generator_id_epa', sa.Text(), nullable=True, comment='Generator ID used by the EPA.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('boiler_id', sa.Text(), nullable=True, comment='Alphanumeric boiler ID.'), + sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['core_eia__entity_boilers.plant_id_eia', 'core_eia__entity_boilers.boiler_id'], name=op.f('fk_core_epa__assn_eia_epacamd_plant_id_eia_core_eia__entity_boilers')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_core_epa__assn_eia_epacamd_plant_id_eia_core_eia__entity_generators')) + ) + op.create_table('core_ferc1__yearly_balance_sheet_assets_sched110', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('amount_type', sa.Text(), nullable=False, comment='Label describing the type of amount being reported. This could be a balance or a change in value.'), - sa.Column('amount', sa.Float(), nullable=True, comment='Reported amount of dollars. This could be a balance or a change in value.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('asset_type', sa.Text(), nullable=False, comment='Type of asset being reported to the core_ferc1__yearly_balance_sheet_assets_sched110 table.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), + sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_cash_flow_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_cash_flow_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'amount_type', name=op.f('pk_denorm_cash_flow_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_balance_sheet_assets_sched110_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'asset_type', name=op.f('pk_core_ferc1__yearly_balance_sheet_assets_sched110')) ) - op.create_table('denorm_depreciation_amortization_summary_ferc1', + op.create_table('core_ferc1__yearly_balance_sheet_liabilities_sched110', sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('plant_function', sa.Text(), nullable=False, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'), - sa.Column('ferc_account_label', sa.Text(), nullable=False, comment='Long FERC account identifier derived from values reported in the XBRL taxonomies. May also refer to aggregations of individual FERC accounts.'), + sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), + sa.Column('liability_type', sa.Text(), nullable=False, comment='Type of liability being reported to the core_ferc1__yearly_balance_sheet_liabilities_sched110 table.'), + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_depreciation_amortization_summary_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_depreciation_amortization_summary_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'ferc_account_label', name=op.f('pk_denorm_depreciation_amortization_summary_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_balance_sheet_liabilities_sched110_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'liability_type', name=op.f('pk_core_ferc1__yearly_balance_sheet_liabilities_sched110')) ) - op.create_table('denorm_electric_energy_dispositions_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + op.create_table('core_ferc1__yearly_cash_flows_sched120', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('energy_disposition_type', sa.Text(), nullable=False, comment='Type of energy disposition reported in the electric_energy_dispositions_ferc1. Dispositions include sales to customers, re-sales of energy, energy used internally, losses, etc.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('amount_type', sa.Text(), nullable=False, comment='Label describing the type of amount being reported. This could be a balance or a change in value.'), + sa.Column('amount', sa.Float(), nullable=True, comment='Reported amount of dollars. This could be a balance or a change in value.'), + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_electric_energy_dispositions_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_electric_energy_dispositions_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_disposition_type', name=op.f('pk_denorm_electric_energy_dispositions_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_cash_flows_sched120_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'amount_type', name=op.f('pk_core_ferc1__yearly_cash_flows_sched120')) ) - op.create_table('denorm_electric_energy_sources_ferc1', + op.create_table('core_ferc1__yearly_depreciation_by_function_sched219', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('energy_source_type', sa.Text(), nullable=False, comment='Type of energy source reported in the electric_energy_sources_ferc1 table. There are three groups of energy sources: generation, power exchanges and transmission.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), + sa.Column('depreciation_type', sa.Text(), nullable=True, comment='Type of depreciation provision within FERC Account 108, including cost ofremoval, depreciation expenses, salvage, cost of retired plant, etc.'), + sa.Column('plant_function', sa.Text(), nullable=False, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'), + sa.Column('plant_status', sa.Text(), nullable=False, comment='Utility plant financial status (in service, future, leased, total).'), + sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_electric_energy_sources_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_electric_energy_sources_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_source_type', name=op.f('pk_denorm_electric_energy_sources_ferc1')) + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_depreciation_by_function_sched219_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'plant_status', 'utility_type', name=op.f('pk_core_ferc1__yearly_depreciation_by_function_sched219')) ) - op.create_table('denorm_electric_operating_expenses_ferc1', + op.create_table('core_ferc1__yearly_depreciation_changes_sched219', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('depreciation_type', sa.Text(), nullable=False, comment='Type of depreciation provision within FERC Account 108, including cost ofremoval, depreciation expenses, salvage, cost of retired plant, etc.'), + sa.Column('plant_status', sa.Text(), nullable=False, comment='Utility plant financial status (in service, future, leased, total).'), + sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('expense_type', sa.Text(), nullable=False, comment='The type of expense.'), - sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_electric_operating_expenses_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_electric_operating_expenses_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'expense_type', name=op.f('pk_denorm_electric_operating_expenses_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_depreciation_changes_sched219_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'depreciation_type', 'plant_status', 'utility_type', name=op.f('pk_core_ferc1__yearly_depreciation_changes_sched219')) ) - op.create_table('denorm_electric_operating_revenues_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + op.create_table('core_ferc1__yearly_depreciation_summary_sched336', sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('revenue_type', sa.Text(), nullable=False, comment='Label describing types of revenues.'), - sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), - sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('plant_function', sa.Text(), nullable=False, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'), + sa.Column('ferc_account_label', sa.Text(), nullable=False, comment='Long FERC account identifier derived from values reported in the XBRL taxonomies. May also refer to aggregations of individual FERC accounts.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_electric_operating_revenues_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_electric_operating_revenues_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'revenue_type', name=op.f('pk_denorm_electric_operating_revenues_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_depreciation_summary_sched336_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'ferc_account_label', name=op.f('pk_core_ferc1__yearly_depreciation_summary_sched336')) ) - op.create_table('denorm_electric_plant_depreciation_changes_ferc1', + op.create_table('core_ferc1__yearly_energy_dispositions_sched401', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('depreciation_type', sa.Text(), nullable=False, comment='Type of depreciation provision within FERC Account 108, including cost ofremoval, depreciation expenses, salvage, cost of retired plant, etc.'), - sa.Column('plant_status', sa.Text(), nullable=False, comment='Utility plant financial status (in service, future, leased, total).'), - sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), + sa.Column('energy_disposition_type', sa.Text(), nullable=False, comment='Type of energy disposition reported in the core_ferc1__yearly_energy_dispositions_sched401. Dispositions include sales to customers, re-sales of energy, energy used internally, losses, etc.'), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_energy_dispositions_sched401_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_disposition_type', name=op.f('pk_core_ferc1__yearly_energy_dispositions_sched401')) + ) + op.create_table('core_ferc1__yearly_energy_sources_sched401', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('energy_source_type', sa.Text(), nullable=False, comment='Type of energy source reported in the core_ferc1__yearly_energy_sources_sched401 table. There are three groups of energy sources: generation, power exchanges and transmission.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_electric_plant_depreciation_changes_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_electric_plant_depreciation_changes_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'depreciation_type', 'plant_status', 'utility_type', name=op.f('pk_denorm_electric_plant_depreciation_changes_ferc1')) + sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_energy_sources_sched401_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_source_type', name=op.f('pk_core_ferc1__yearly_energy_sources_sched401')) ) - op.create_table('denorm_electric_plant_depreciation_functional_ferc1', + op.create_table('core_ferc1__yearly_income_statements_sched114', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('depreciation_type', sa.Text(), nullable=True, comment='Type of depreciation provision within FERC Account 108, including cost ofremoval, depreciation expenses, salvage, cost of retired plant, etc.'), - sa.Column('plant_function', sa.Text(), nullable=False, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'), - sa.Column('plant_status', sa.Text(), nullable=False, comment='Utility plant financial status (in service, future, leased, total).'), sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('income_type', sa.Text(), nullable=False, comment='Type of income reported in income_statement_ferc1 table.'), + sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_electric_plant_depreciation_functional_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_electric_plant_depreciation_functional_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'plant_status', 'utility_type', name=op.f('pk_denorm_electric_plant_depreciation_functional_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_income_statements_sched114_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'income_type', name=op.f('pk_core_ferc1__yearly_income_statements_sched114')) ) - op.create_table('denorm_electricity_sales_by_rate_schedule_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('rate_schedule_type', sa.Text(), nullable=True, comment='Categorization of rate schedule type.'), - sa.Column('billing_status', sa.Text(), nullable=True, comment='Whether an amount is billed, unbilled, or both. Billed amounts pertain to the exchange of energy and unbilled amounts pertain to other sources of revenue such as contracts with peaker plants to keep them on standby or charging rent to host cell antennas on transmission towers.'), - sa.Column('rate_schedule_description', sa.Text(), nullable=True, comment='Free-form description of what the rate schedule name is. Not standardized. Often a sub-category of rate_schedule_type.'), - sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), + op.create_table('core_ferc1__yearly_operating_expenses_sched320', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), - sa.Column('kwh_per_customer', sa.Float(), nullable=True, comment='kwh per customer.'), - sa.Column('revenue_per_kwh', sa.Float(), nullable=True), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_electricity_sales_by_rate_schedule_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_electricity_sales_by_rate_schedule_ferc1_utility_id_pudl_utilities_pudl')) - ) - op.create_table('denorm_generation_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_generation_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_denorm_generation_eia923_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_generation_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_generation_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_generation_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_denorm_generation_eia923')) - ) - op.create_table('denorm_generation_monthly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_denorm_generation_monthly_eia923_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_generation_monthly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_denorm_generation_monthly_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_generation_monthly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_denorm_generation_monthly_eia923')) - ) - op.create_table('denorm_income_statement_ferc1', + sa.Column('expense_type', sa.Text(), nullable=False, comment='The type of expense.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_operating_expenses_sched320_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'expense_type', name=op.f('pk_core_ferc1__yearly_operating_expenses_sched320')) + ) + op.create_table('core_ferc1__yearly_operating_revenues_sched300', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('income_type', sa.Text(), nullable=False, comment='Type of income reported in income_statement_ferc1 table.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('revenue_type', sa.Text(), nullable=False, comment='Label describing types of revenues.'), sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), + sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), + sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_income_statement_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_income_statement_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'income_type', name=op.f('pk_denorm_income_statement_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_operating_revenues_sched300_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'revenue_type', name=op.f('pk_core_ferc1__yearly_operating_revenues_sched300')) ) - op.create_table('denorm_other_regulatory_liabilities_ferc1', + op.create_table('core_ferc1__yearly_other_regulatory_liabilities_sched278', sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), @@ -1471,58 +1456,50 @@ def upgrade() -> None: sa.Column('increase_in_other_regulatory_liabilities', sa.Float(), nullable=True, comment='The increase during the reporting period of other regulatory liabilities.'), sa.Column('account_detail', sa.Text(), nullable=True, comment='Description of the account number credited from making debit adjustment to other regulatory liabilities.'), sa.Column('decrease_in_other_regulatory_liabilities', sa.Float(), nullable=True, comment='The decrease during the reporting period of other regulatory liabilities.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_other_regulatory_liabilities_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_other_regulatory_liabilities_ferc1_utility_id_pudl_utilities_pudl')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_other_regulatory_liabilities_sched278_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')) ) - op.create_table('denorm_plant_in_service_ferc1', - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + op.create_table('core_ferc1__yearly_plant_in_service_sched204', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('additions', sa.Float(), nullable=True, comment='Cost of acquisition of items classified within the account.'), - sa.Column('adjustments', sa.Float(), nullable=True, comment='Cost of adjustments to the account.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('ferc_account_label', sa.Text(), nullable=False, comment='Long FERC account identifier derived from values reported in the XBRL taxonomies. May also refer to aggregations of individual FERC accounts.'), - sa.Column('retirements', sa.Float(), nullable=True, comment='Cost of disposal of items classified within the account.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), + sa.Column('additions', sa.Float(), nullable=True, comment='Cost of acquisition of items classified within the account.'), + sa.Column('retirements', sa.Float(), nullable=True, comment='Cost of disposal of items classified within the account.'), + sa.Column('adjustments', sa.Float(), nullable=True, comment='Cost of adjustments to the account.'), sa.Column('transfers', sa.Float(), nullable=True, comment='Cost of transfers into (out of) the account.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_plant_in_service_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plant_in_service_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'ferc_account_label', name=op.f('pk_denorm_plant_in_service_ferc1')) + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('plant_status', sa.Text(), nullable=True, comment='Utility plant financial status (in service, future, leased, total).'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_plant_in_service_sched204_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'ferc_account_label', name=op.f('pk_core_ferc1__yearly_plant_in_service_sched204')) ) - op.create_table('denorm_purchased_power_ferc1', - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + op.create_table('core_ferc1__yearly_purchased_power_and_exchanges_sched326', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('seller_name', sa.Text(), nullable=True, comment='Name of the seller, or the other party in an exchange transaction.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('purchase_type_code', sa.Text(), nullable=True, comment="Categorization based on the original contractual terms and conditions of the service. Must be one of 'requirements', 'long_firm', 'intermediate_firm', 'short_firm', 'long_unit', 'intermediate_unit', 'electricity_exchange', 'other_service', or 'adjustment'. Requirements service is ongoing high reliability service, with load integrated into system resource planning. 'Long term' means 5+ years. 'Intermediate term' is 1-5 years. 'Short term' is less than 1 year. 'Firm' means not interruptible for economic reasons. 'unit' indicates service from a particular designated generating unit. 'exchange' is an in-kind transaction."), + sa.Column('tariff', sa.Text(), nullable=True, comment='FERC Rate Schedule Number or Tariff. (Note: may be incomplete if originally reported on multiple lines.)'), sa.Column('billing_demand_mw', sa.Float(), nullable=True, comment='Monthly average billing demand (for requirements purchases, and any transactions involving demand charges). In megawatts.'), - sa.Column('coincident_peak_demand_mw', sa.Float(), nullable=True, comment="Average monthly coincident peak (CP) demand (for requirements purchases, and any transactions involving demand charges). Monthly CP demand is the metered demand during the hour (60-minute integration) in which the supplier's system reaches its monthly peak. In megawatts."), - sa.Column('delivered_mwh', sa.Float(), nullable=True, comment='Gross megawatt-hours delivered in power exchanges and used as the basis for settlement.'), - sa.Column('demand_charges', sa.Float(), nullable=True, comment='Demand charges (USD).'), - sa.Column('energy_charges', sa.Float(), nullable=True, comment='Energy charges (USD).'), sa.Column('non_coincident_peak_demand_mw', sa.Float(), nullable=True, comment='Average monthly non-coincident peak (NCP) demand (for requirements purhcases, and any transactions involving demand charges). Monthly NCP demand is the maximum metered hourly (60-minute integration) demand in a month. In megawatts.'), - sa.Column('other_charges', sa.Float(), nullable=True, comment='Other charges, including out-of-period adjustments (USD).'), - sa.Column('purchase_type_code', sa.Text(), nullable=True, comment="Categorization based on the original contractual terms and conditions of the service. Must be one of 'requirements', 'long_firm', 'intermediate_firm', 'short_firm', 'long_unit', 'intermediate_unit', 'electricity_exchange', 'other_service', or 'adjustment'. Requirements service is ongoing high reliability service, with load integrated into system resource planning. 'Long term' means 5+ years. 'Intermediate term' is 1-5 years. 'Short term' is less than 1 year. 'Firm' means not interruptible for economic reasons. 'unit' indicates service from a particular designated generating unit. 'exchange' is an in-kind transaction."), + sa.Column('coincident_peak_demand_mw', sa.Float(), nullable=True, comment="Average monthly coincident peak (CP) demand (for requirements purchases, and any transactions involving demand charges). Monthly CP demand is the metered demand during the hour (60-minute integration) in which the supplier's system reaches its monthly peak. In megawatts."), sa.Column('purchased_mwh', sa.Float(), nullable=True, comment='Megawatt-hours shown on bills rendered to the respondent. Includes both electricity purchased for storage and non-storage purposes, which were lumped together prior to 2021.'), sa.Column('purchased_storage_mwh', sa.Float(), nullable=True, comment='Number of megawatt hours purchased during the period for energy storage.'), sa.Column('purchased_other_than_storage_mwh', sa.Float(), nullable=True, comment='Number of megawatt hours purchased during the period for other than energy storage.'), sa.Column('received_mwh', sa.Float(), nullable=True, comment='Gross megawatt-hours received in power exchanges and used as the basis for settlement.'), - sa.Column('tariff', sa.Text(), nullable=True, comment='FERC Rate Schedule Number or Tariff. (Note: may be incomplete if originally reported on multiple lines.)'), + sa.Column('delivered_mwh', sa.Float(), nullable=True, comment='Gross megawatt-hours delivered in power exchanges and used as the basis for settlement.'), + sa.Column('demand_charges', sa.Float(), nullable=True, comment='Demand charges (USD).'), + sa.Column('energy_charges', sa.Float(), nullable=True, comment='Energy charges (USD).'), + sa.Column('other_charges', sa.Float(), nullable=True, comment='Other charges, including out-of-period adjustments (USD).'), sa.Column('total_settlement', sa.Float(), nullable=True, comment='Sum of demand, energy, and other charges (USD). For power exchanges, the settlement amount for the net receipt of energy. If more energy was delivered than received, this amount is negative.'), - sa.ForeignKeyConstraint(['purchase_type_code'], ['power_purchase_types_ferc1.code'], name=op.f('fk_denorm_purchased_power_ferc1_purchase_type_code_power_purchase_types_ferc1')), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_purchased_power_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_purchased_power_ferc1_utility_id_pudl_utilities_pudl')) + sa.ForeignKeyConstraint(['purchase_type_code'], ['core_ferc1__codes_power_purchase_types.code'], name=op.f('fk_core_ferc1__yearly_purchased_power_and_exchanges_sched326_purchase_type_code_core_ferc1__codes_power_purchase_types')), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_purchased_power_and_exchanges_sched326_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')) ) - op.create_table('denorm_retained_earnings_ferc1', + op.create_table('core_ferc1__yearly_retained_earnings_sched118', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('earnings_type', sa.Text(), nullable=False, comment='Label describing types of earnings.'), @@ -1531,15 +1508,27 @@ def upgrade() -> None: sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_retained_earnings_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_retained_earnings_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'earnings_type', name=op.f('pk_denorm_retained_earnings_ferc1')) + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_retained_earnings_sched118_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'earnings_type', name=op.f('pk_core_ferc1__yearly_retained_earnings_sched118')) + ) + op.create_table('core_ferc1__yearly_sales_by_rate_schedules_sched304', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('rate_schedule_type', sa.Text(), nullable=True, comment='Categorization of rate schedule type.'), + sa.Column('billing_status', sa.Text(), nullable=True, comment='Whether an amount is billed, unbilled, or both. Billed amounts pertain to the exchange of energy and unbilled amounts pertain to other sources of revenue such as contracts with peaker plants to keep them on standby or charging rent to host cell antennas on transmission towers.'), + sa.Column('rate_schedule_description', sa.Text(), nullable=True, comment='Free-form description of what the rate schedule name is. Not standardized. Often a sub-category of rate_schedule_type.'), + sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), + sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), + sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), + sa.Column('kwh_per_customer', sa.Float(), nullable=True, comment='kwh per customer.'), + sa.Column('revenue_per_kwh', sa.Float(), nullable=True), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_sales_by_rate_schedules_sched304_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')) ) - op.create_table('denorm_transmission_statistics_ferc1', + op.create_table('core_ferc1__yearly_transmission_lines_sched422', sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('start_point', sa.Text(), nullable=True, comment='The starting point of a transmission line.'), sa.Column('end_point', sa.Text(), nullable=True, comment='The end point of a transmission line.'), @@ -1550,17 +1539,314 @@ def upgrade() -> None: sa.Column('transmission_line_and_structures_length_miles', sa.Float(), nullable=True, comment='Length (in pole miles or circuit miles (if transmission lines are underground)) for lines that are agrregated with other lines / structures (whose cost are aggregated and combined with other structures).'), sa.Column('num_transmission_circuits', sa.Integer(), nullable=True, comment='Number of circuits in a transmission line.'), sa.Column('conductor_size_and_material', sa.Text(), nullable=True, comment='Size of transmission conductor and material of the transmission line.'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), - sa.Column('capex_other', sa.Float(), nullable=True, comment='Other costs associated with the plant (USD).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_transmission_statistics_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_transmission_statistics_ferc1_utility_id_pudl_utilities_pudl')) + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of Land and land rights for the transmission line.'), + sa.Column('capex_other', sa.Float(), nullable=True, comment='Construction and other costs for the transmission line.'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total costs for the transmission line.'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Operating expenses for the transmission line.'), + sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Maintenance expenses for the transmission line.'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Rent expenses for the transmission line.'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Overall expenses for the transmission line.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_transmission_lines_sched422_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')) + ) + op.create_table('core_ferc1__yearly_utility_plant_summary_sched200', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('utility_type_other', sa.Text(), nullable=True, comment='Freeform description of type of utility reported in one of the other three other utility_type sections in the core_ferc1__yearly_utility_plant_summary_sched200 table. This field is reported only in the DBF reporting years (1994-2020).'), + sa.Column('utility_plant_asset_type', sa.Text(), nullable=False, comment='Type of utility plant asset reported in the core_ferc1__yearly_utility_plant_summary_sched200 table. Assets include those leased to others, held for future use, construction work-in-progress and details of accumulated depreciation.'), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_ferc1__yearly_utility_plant_summary_sched200_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'utility_plant_asset_type', name=op.f('pk_core_ferc1__yearly_utility_plant_summary_sched200')) + ) + op.create_table('core_pudl__assn_ferc1_dbf_pudl_utilities', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_ferc1_dbf', sa.Integer(), nullable=False, comment='FERC-assigned respondent_id from DBF reporting years, identifying the reporting entity. Stable from year to year.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_pudl__assn_ferc1_dbf_pudl_utilities_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1_dbf', name=op.f('pk_core_pudl__assn_ferc1_dbf_pudl_utilities')) + ) + op.create_table('core_pudl__assn_ferc1_pudl_plants', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=False, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_core_pudl__assn_ferc1_pudl_plants_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_pudl__assn_ferc1_pudl_plants_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'plant_name_ferc1', name=op.f('pk_core_pudl__assn_ferc1_pudl_plants')) + ) + op.create_table('core_pudl__assn_ferc1_xbrl_pudl_utilities', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_ferc1_xbrl', sa.Text(), nullable=False, comment='FERC-assigned entity_id from XBRL reporting years, identifying the reporting entity. Stable from year to year.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_core_pudl__assn_ferc1_xbrl_pudl_utilities_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.PrimaryKeyConstraint('utility_id_ferc1_xbrl', name=op.f('pk_core_pudl__assn_ferc1_xbrl_pudl_utilities')) + ) + op.create_table('out_eia923__boiler_fuel', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), + sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__boiler_fuel_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__boiler_fuel_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['core_eia__entity_boilers.plant_id_eia', 'core_eia__entity_boilers.boiler_id'], name=op.f('fk_out_eia923__boiler_fuel_plant_id_eia_core_eia__entity_boilers')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__boiler_fuel_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__boiler_fuel_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__boiler_fuel_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__boiler_fuel_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_out_eia923__boiler_fuel')) + ) + op.create_table('out_eia923__generation', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__generation_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_out_eia923__generation_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__generation_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__generation_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__generation_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_out_eia923__generation')) + ) + op.create_table('out_eia923__monthly_boiler_fuel', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), + sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__monthly_boiler_fuel_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__monthly_boiler_fuel_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['core_eia__entity_boilers.plant_id_eia', 'core_eia__entity_boilers.boiler_id'], name=op.f('fk_out_eia923__monthly_boiler_fuel_plant_id_eia_core_eia__entity_boilers')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__monthly_boiler_fuel_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__monthly_boiler_fuel_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__monthly_boiler_fuel_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__monthly_boiler_fuel_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_out_eia923__monthly_boiler_fuel')) + ) + op.create_table('out_eia923__monthly_generation', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__monthly_generation_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_out_eia923__monthly_generation_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__monthly_generation_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__monthly_generation_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__monthly_generation_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_out_eia923__monthly_generation')) + ) + op.create_table('out_eia923__monthly_generation_fuel_by_generator', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_out_eia923__monthly_generation_fuel_by_generator_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__monthly_generation_fuel_by_generator_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia923__monthly_generation_fuel_by_generator_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__monthly_generation_fuel_by_generator_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_out_eia923__monthly_generation_fuel_by_generator')) + ) + op.create_table('out_eia923__monthly_generation_fuel_by_generator_energy_source', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('energy_source_code_num', sa.Enum('energy_source_code_1', 'energy_source_code_2', 'energy_source_code_3', 'energy_source_code_4', 'energy_source_code_5', 'energy_source_code_6', 'energy_source_code_7', 'energy_source_code_8'), nullable=True, comment='Name of the energy_source_code_N column that this energy source code was reported in for the generator referenced in the same record.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__monthly_generation_fuel_by_generator_energy_source_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_out_eia923__monthly_generation_fuel_by_generator_energy_source_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__monthly_generation_fuel_by_generator_energy_source_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'prime_mover_code', 'energy_source_code', name=op.f('pk_out_eia923__monthly_generation_fuel_by_generator_energy_source')) ) - op.create_table('denorm_utilities_eia', + op.create_table('out_eia__monthly_generators', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), + sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), + sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), + sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), + sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), + sa.Column('associated_combined_heat_power', sa.Boolean(), nullable=True, comment='Indicates whether the generator is associated with a combined heat and power system'), + sa.Column('bga_source', sa.Text(), nullable=True, comment='The source from where the unit_id_pudl is compiled. The unit_id_pudl comes directly from EIA 860, or string association (which looks at all the boilers and generators that are not associated with a unit and tries to find a matching string in the respective collection of boilers or generator), or from a unit connection (where the unit_id_eia is employed to find additional boiler generator connections).'), + sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), + sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), + sa.Column('county', sa.Text(), nullable=True, comment='County name.'), + sa.Column('current_planned_generator_operating_date', sa.Date(), nullable=True, comment='The most recently updated effective date on which the generator is scheduled to start operation'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.Column('deliver_power_transgrid', sa.Boolean(), nullable=True, comment='Indicate whether the generator can deliver power to the transmission grid.'), + sa.Column('distributed_generation', sa.Boolean(), nullable=True, comment='Whether the generator is considered distributed generation'), + sa.Column('duct_burners', sa.Boolean(), nullable=True, comment='Indicates whether the unit has duct-burners for supplementary firing of the turbine exhaust gas'), + sa.Column('energy_source_1_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 1.'), + sa.Column('energy_source_1_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 1.'), + sa.Column('energy_source_1_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 1.'), + sa.Column('energy_source_2_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 2.'), + sa.Column('energy_source_2_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 2.'), + sa.Column('energy_source_2_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 2.'), + sa.Column('energy_source_code_2', sa.Text(), nullable=True, comment='The code representing the second most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_3', sa.Text(), nullable=True, comment='The code representing the third most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_4', sa.Text(), nullable=True, comment='The code representing the fourth most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_5', sa.Text(), nullable=True, comment='The code representing the fifth most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_6', sa.Text(), nullable=True, comment='The code representing the sixth most predominant type of energy that fuels the generator'), + sa.Column('energy_storage_capacity_mwh', sa.Float(), nullable=True, comment='Energy storage capacity in MWh (e.g. for batteries).'), + sa.Column('ferc_qualifying_facility', sa.Boolean(), nullable=True, comment='Indicatates whether or not a generator is a qualifying FERC cogeneation facility.'), + sa.Column('fluidized_bed_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses fluidized bed technology'), + sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), + sa.Column('latitude', sa.Float(), nullable=True, comment="Latitude of the plant's location, in degrees."), + sa.Column('longitude', sa.Float(), nullable=True, comment="Longitude of the plant's location, in degrees."), + sa.Column('minimum_load_mw', sa.Float(), nullable=True, comment='The minimum load at which the generator can operate at continuosuly.'), + sa.Column('multiple_fuels', sa.Boolean(), nullable=True, comment='Can the generator burn multiple fuels?'), + sa.Column('nameplate_power_factor', sa.Float(), nullable=True, comment='The nameplate power factor of the generator.'), + sa.Column('net_capacity_mwdc', sa.Float(), nullable=True, comment='Generation capacity in megawatts of direct current that is subject to a net metering agreement. Typically used for behind-the-meter solar PV.'), + sa.Column('operating_switch', sa.Text(), nullable=True, comment='Indicates whether the fuel switching generator can switch when operating'), + sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), + sa.Column('original_planned_generator_operating_date', sa.Date(), nullable=True, comment='The date the generator was originally scheduled to be operational'), + sa.Column('other_combustion_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses other combustion technologies'), + sa.Column('other_modifications_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter commercial operation after any other planned modification is complete.'), + sa.Column('other_planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are there other modifications planned for the generator.'), + sa.Column('owned_by_non_utility', sa.Boolean(), nullable=True, comment='Whether any part of generator is owned by a nonutilty'), + sa.Column('ownership_code', sa.Text(), nullable=True, comment='Identifies the ownership for each generator.'), + sa.Column('planned_derate_date', sa.Date(), nullable=True, comment='Planned effective month that the generator is scheduled to enter operation after the derate modification.'), + sa.Column('planned_energy_source_code_1', sa.Text(), nullable=True, comment='New energy source code for the planned repowered generator.'), + sa.Column('planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are any planned capacity uprates/derates, repowering, other modifications, or generator retirements scheduled for the next 5 years.'), + sa.Column('planned_net_summer_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in summer capacity expected to be realized from the derate modification to the equipment.'), + sa.Column('planned_net_summer_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in summer capacity expected to be realized from the modification to the equipment.'), + sa.Column('planned_net_winter_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in winter capacity expected to be realized from the derate modification to the equipment.'), + sa.Column('planned_net_winter_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in winter capacity expected to be realized from the uprate modification to the equipment.'), + sa.Column('planned_new_capacity_mw', sa.Float(), nullable=True, comment='The expected new namplate capacity for the generator.'), + sa.Column('planned_new_prime_mover_code', sa.Text(), nullable=True, comment='New prime mover for the planned repowered generator.'), + sa.Column('planned_repower_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the repowering is complete.'), + sa.Column('planned_uprate_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the uprate modification.'), + sa.Column('previously_canceled', sa.Boolean(), nullable=True, comment='Indicates whether the generator was previously reported as indefinitely postponed or canceled'), + sa.Column('pulverized_coal_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses pulverized coal technology'), + sa.Column('reactive_power_output_mvar', sa.Float(), nullable=True, comment='Reactive Power Output (MVAr)'), + sa.Column('rto_iso_lmp_node_id', sa.Text(), nullable=True, comment='The designation used to identify the price node in RTO/ISO Locational Marginal Price reports'), + sa.Column('rto_iso_location_wholesale_reporting_id', sa.Text(), nullable=True, comment='The designation used to report ths specific location of the wholesale sales transactions to FERC for the Electric Quarterly Report'), + sa.Column('solid_fuel_gasification', sa.Boolean(), nullable=True, comment='Indicates whether the generator is part of a solid fuel gasification system'), + sa.Column('startup_source_code_1', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_2', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_3', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_4', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('stoker_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses stoker technology'), + sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('subcritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses subcritical technology'), + sa.Column('summer_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the summer capacity value was an estimate'), + sa.Column('summer_capacity_mw', sa.Float(), nullable=True, comment='The net summer capacity.'), + sa.Column('summer_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated summer capacity (in MWh).'), + sa.Column('supercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses supercritical technology'), + sa.Column('switch_oil_gas', sa.Boolean(), nullable=True, comment='Indicates whether the generator switch between oil and natural gas.'), + sa.Column('syncronized_transmission_grid', sa.Boolean(), nullable=True, comment='Indicates whether standby generators (SB status) can be synchronized to the grid.'), + sa.Column('time_cold_shutdown_full_load_code', sa.Text(), nullable=True, comment='The minimum amount of time required to bring the unit to full load from shutdown.'), + sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), + sa.Column('topping_bottoming_code', sa.Text(), nullable=True, comment='If the generator is associated with a combined heat and power system, indicates whether the generator is part of a topping cycle or a bottoming cycle'), + sa.Column('turbines_inverters_hydrokinetics', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), + sa.Column('turbines_num', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), + sa.Column('ultrasupercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses ultra-supercritical technology'), + sa.Column('uprate_derate_completed_date', sa.Date(), nullable=True, comment='The date when the uprate or derate was completed.'), + sa.Column('uprate_derate_during_year', sa.Boolean(), nullable=True, comment='Was an uprate or derate completed on this generator during the reporting year?'), + sa.Column('winter_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the winter capacity value was an estimate'), + sa.Column('winter_capacity_mw', sa.Float(), nullable=True, comment='The net winter capacity.'), + sa.Column('winter_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated winter capacity (in MWh).'), + sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia__monthly_generators_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_1_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_1_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_1_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_1_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_2_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_2_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_2_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_5'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_code_5_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_6'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_energy_source_code_6_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['operational_status_code'], ['core_eia__codes_operational_status.code'], name=op.f('fk_out_eia__monthly_generators_operational_status_code_core_eia__codes_operational_status')), + sa.ForeignKeyConstraint(['planned_energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_planned_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['planned_new_prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia__monthly_generators_planned_new_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_out_eia__monthly_generators_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia__monthly_generators_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia__monthly_generators_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['startup_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_startup_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_startup_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_startup_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__monthly_generators_startup_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['utility_id_eia'], ['core_eia__entity_utilities.utility_id_eia'], name=op.f('fk_out_eia__monthly_generators_utility_id_eia_core_eia__entity_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia__monthly_generators_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_out_eia__monthly_generators')) + ) + op.create_table('out_eia__yearly_utilities', sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), @@ -1588,87 +1874,80 @@ def upgrade() -> None: sa.Column('phone_number_2', sa.Text(), nullable=True, comment='Phone number for utility contact 2.'), sa.Column('phone_extension_2', sa.Text(), nullable=True, comment='Phone extension for utility contact 2'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_utilities_eia_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_utilities_eia_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_utilities_eia_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_eia', 'report_date', name=op.f('pk_denorm_utilities_eia')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia__yearly_utilities_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia__yearly_utilities_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia__yearly_utilities_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_eia', 'report_date', name=op.f('pk_out_eia__yearly_utilities')) ) - op.create_table('denorm_utility_plant_summary_ferc1', + op.create_table('out_ferc1__yearly_balance_sheet_assets_sched110', + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('utility_type_other', sa.Text(), nullable=True, comment='Freeform description of type of utility reported in one of the other three other utility_type sections in the utility_plant_summary_ferc1 table. This field is reported only in the DBF reporting years (1994-2020).'), - sa.Column('utility_plant_asset_type', sa.Text(), nullable=False, comment='Type of utility plant asset reported in the utility_plant_summary_ferc1 table. Assets include those leased to others, held for future use, construction work-in-progress and details of accumulated depreciation.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_denorm_utility_plant_summary_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_utility_plant_summary_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'utility_plant_asset_type', name=op.f('pk_denorm_utility_plant_summary_ferc1')) + sa.Column('asset_type', sa.Text(), nullable=False, comment='Type of asset being reported to the core_ferc1__yearly_balance_sheet_assets_sched110 table.'), + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_balance_sheet_assets_sched110_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_balance_sheet_assets_sched110_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'asset_type', name=op.f('pk_out_ferc1__yearly_balance_sheet_assets_sched110')) ) - op.create_table('depreciation_amortization_summary_ferc1', + op.create_table('out_ferc1__yearly_balance_sheet_liabilities_sched110', sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('plant_function', sa.Text(), nullable=False, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'), - sa.Column('ferc_account_label', sa.Text(), nullable=False, comment='Long FERC account identifier derived from values reported in the XBRL taxonomies. May also refer to aggregations of individual FERC accounts.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), + sa.Column('liability_type', sa.Text(), nullable=False, comment='Type of liability being reported to the core_ferc1__yearly_balance_sheet_liabilities_sched110 table.'), + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), - sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_depreciation_amortization_summary_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'ferc_account_label', name=op.f('pk_depreciation_amortization_summary_ferc1')) + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_balance_sheet_liabilities_sched110_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_balance_sheet_liabilities_sched110_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'liability_type', name=op.f('pk_out_ferc1__yearly_balance_sheet_liabilities_sched110')) ) - op.create_table('electric_energy_dispositions_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('energy_disposition_type', sa.Text(), nullable=False, comment='Type of energy disposition reported in the electric_energy_dispositions_ferc1. Dispositions include sales to customers, re-sales of energy, energy used internally, losses, etc.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), + op.create_table('out_ferc1__yearly_cash_flows_sched120', sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_electric_energy_dispositions_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_disposition_type', name=op.f('pk_electric_energy_dispositions_ferc1')) - ) - op.create_table('electric_energy_sources_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('energy_source_type', sa.Text(), nullable=False, comment='Type of energy source reported in the electric_energy_sources_ferc1 table. There are three groups of energy sources: generation, power exchanges and transmission.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_electric_energy_sources_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_source_type', name=op.f('pk_electric_energy_sources_ferc1')) - ) - op.create_table('electric_operating_expenses_ferc1', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('expense_type', sa.Text(), nullable=False, comment='The type of expense.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('amount_type', sa.Text(), nullable=False, comment='Label describing the type of amount being reported. This could be a balance or a change in value.'), + sa.Column('amount', sa.Float(), nullable=True, comment='Reported amount of dollars. This could be a balance or a change in value.'), + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_electric_operating_expenses_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'expense_type', name=op.f('pk_electric_operating_expenses_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_cash_flows_sched120_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_cash_flows_sched120_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'amount_type', name=op.f('pk_out_ferc1__yearly_cash_flows_sched120')) ) - op.create_table('electric_operating_revenues_ferc1', + op.create_table('out_ferc1__yearly_depreciation_by_function_sched219', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('depreciation_type', sa.Text(), nullable=True, comment='Type of depreciation provision within FERC Account 108, including cost ofremoval, depreciation expenses, salvage, cost of retired plant, etc.'), + sa.Column('plant_function', sa.Text(), nullable=False, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'), + sa.Column('plant_status', sa.Text(), nullable=False, comment='Utility plant financial status (in service, future, leased, total).'), + sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('revenue_type', sa.Text(), nullable=False, comment='Label describing types of revenues.'), - sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), - sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), - sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), - sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_electric_operating_revenues_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'revenue_type', name=op.f('pk_electric_operating_revenues_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_depreciation_by_function_sched219_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_depreciation_by_function_sched219_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'plant_status', 'utility_type', name=op.f('pk_out_ferc1__yearly_depreciation_by_function_sched219')) ) - op.create_table('electric_plant_depreciation_changes_ferc1', + op.create_table('out_ferc1__yearly_depreciation_changes_sched219', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('depreciation_type', sa.Text(), nullable=False, comment='Type of depreciation provision within FERC Account 108, including cost ofremoval, depreciation expenses, salvage, cost of retired plant, etc.'), sa.Column('plant_status', sa.Text(), nullable=False, comment='Utility plant financial status (in service, future, leased, total).'), @@ -1678,161 +1957,56 @@ def upgrade() -> None: sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_electric_plant_depreciation_changes_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'depreciation_type', 'plant_status', 'utility_type', name=op.f('pk_electric_plant_depreciation_changes_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_depreciation_changes_sched219_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_depreciation_changes_sched219_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'depreciation_type', 'plant_status', 'utility_type', name=op.f('pk_out_ferc1__yearly_depreciation_changes_sched219')) ) - op.create_table('electric_plant_depreciation_functional_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + op.create_table('out_ferc1__yearly_depreciation_summary_sched336', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('depreciation_type', sa.Text(), nullable=True, comment='Type of depreciation provision within FERC Account 108, including cost ofremoval, depreciation expenses, salvage, cost of retired plant, etc.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('plant_function', sa.Text(), nullable=False, comment='Functional role played by utility plant (steam production, nuclear production, distribution, transmission, etc.).'), - sa.Column('plant_status', sa.Text(), nullable=False, comment='Utility plant financial status (in service, future, leased, total).'), - sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_electric_plant_depreciation_functional_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'plant_status', 'utility_type', name=op.f('pk_electric_plant_depreciation_functional_ferc1')) - ) - op.create_table('electricity_sales_by_rate_schedule_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('rate_schedule_type', sa.Text(), nullable=True, comment='Categorization of rate schedule type.'), - sa.Column('billing_status', sa.Text(), nullable=True, comment='Whether an amount is billed, unbilled, or both. Billed amounts pertain to the exchange of energy and unbilled amounts pertain to other sources of revenue such as contracts with peaker plants to keep them on standby or charging rent to host cell antennas on transmission towers.'), - sa.Column('rate_schedule_description', sa.Text(), nullable=True, comment='Free-form description of what the rate schedule name is. Not standardized. Often a sub-category of rate_schedule_type.'), - sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), + sa.Column('ferc_account_label', sa.Text(), nullable=False, comment='Long FERC account identifier derived from values reported in the XBRL taxonomies. May also refer to aggregations of individual FERC accounts.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), - sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), - sa.Column('kwh_per_customer', sa.Float(), nullable=True, comment='kwh per customer.'), - sa.Column('revenue_per_kwh', sa.Float(), nullable=True), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_electricity_sales_by_rate_schedule_ferc1_utility_id_ferc1_utilities_ferc1')) - ) - op.create_table('epacamd_eia', - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_id_epa', sa.Integer(), nullable=True, comment='The ORISPL ID used by EPA to refer to the plant. Usually but not always the same as plant_id_eia.'), - sa.Column('emissions_unit_id_epa', sa.Text(), nullable=True, comment='Emissions (smokestack) unit monitored by EPA CEMS.'), - sa.Column('generator_id_epa', sa.Text(), nullable=True, comment='Generator ID used by the EPA.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('boiler_id', sa.Text(), nullable=True, comment='Alphanumeric boiler ID.'), - sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['boilers_entity_eia.plant_id_eia', 'boilers_entity_eia.boiler_id'], name=op.f('fk_epacamd_eia_plant_id_eia_boilers_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_epacamd_eia_plant_id_eia_generators_entity_eia')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_depreciation_summary_sched336_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_depreciation_summary_sched336_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'plant_function', 'ferc_account_label', name=op.f('pk_out_ferc1__yearly_depreciation_summary_sched336')) ) - op.create_table('fuel_cost_by_generator_monthly', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + op.create_table('out_ferc1__yearly_energy_dispositions_sched401', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_fuel_cost_by_generator_monthly_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_fuel_cost_by_generator_monthly_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_fuel_cost_by_generator_monthly_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_fuel_cost_by_generator_monthly_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_fuel_cost_by_generator_monthly')) - ) - op.create_table('fuel_receipts_costs_eia923', - sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('contract_type_code', sa.Enum('S', 'C', 'NC', 'T'), nullable=True, comment='Purchase type under which receipts occurred in the reporting month. C: Contract, NC: New Contract, S: Spot Purchase, T: Tolling Agreement.'), - sa.Column('contract_expiration_date', sa.Date(), nullable=True, comment='Date contract expires.Format: MMYY.'), - sa.Column('energy_source_code', sa.Text(), nullable=True, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_group_code', sa.Enum('petroleum', 'other_gas', 'petroleum_coke', 'natural_gas', 'coal'), nullable=True, comment='Fuel groups used in the Electric Power Monthly'), - sa.Column('mine_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL mine identifier.'), - sa.Column('supplier_name', sa.Text(), nullable=True, comment='Company that sold the fuel to the plant or, in the case of Natural Gas, pipline owner.'), - sa.Column('fuel_received_units', sa.Float(), nullable=True, comment='Quanity of fuel received in tons, barrel, or Mcf.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), - sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), - sa.Column('mercury_content_ppm', sa.Float(), nullable=True, comment='Mercury content in parts per million (ppm) to the nearest 0.001 ppm.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('primary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the longest distance transported.'), - sa.Column('secondary_transportation_mode_code', sa.Text(), nullable=True, comment='Transportation mode for the second longest distance transported.'), - sa.Column('natural_gas_transport_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natural gas transportation service.'), - sa.Column('natural_gas_delivery_contract_type_code', sa.Enum('firm', 'interruptible'), nullable=True, comment='Contract type for natrual gas delivery service:'), - sa.Column('moisture_content_pct', sa.Float(), nullable=True), - sa.Column('chlorine_content_ppm', sa.Float(), nullable=True), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['contract_type_code'], ['contract_types_eia.code'], name=op.f('fk_fuel_receipts_costs_eia923_contract_type_code_contract_types_eia')), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_fuel_receipts_costs_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_fuel_receipts_costs_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['mine_id_pudl'], ['coalmine_eia923.mine_id_pudl'], name=op.f('fk_fuel_receipts_costs_eia923_mine_id_pudl_coalmine_eia923')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_fuel_receipts_costs_eia923_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['primary_transportation_mode_code'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_fuel_receipts_costs_eia923_primary_transportation_mode_code_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['secondary_transportation_mode_code'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_fuel_receipts_costs_eia923_secondary_transportation_mode_code_fuel_transportation_modes_eia')) - ) - op.create_table('generation_eia923', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_generation_eia923_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_generation_eia923_plant_id_eia_generators_entity_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_generation_eia923')) - ) - op.create_table('generation_fuel_by_generator_energy_source_monthly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('energy_source_code_num', sa.Enum('energy_source_code_1', 'energy_source_code_2', 'energy_source_code_3', 'energy_source_code_4', 'energy_source_code_5', 'energy_source_code_6', 'energy_source_code_7', 'energy_source_code_8'), nullable=True, comment='Name of the energy_source_code_N column that this energy source code was reported in for the generator referenced in the same record.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_generation_fuel_by_generator_energy_source_monthly_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_generation_fuel_by_generator_energy_source_monthly_eia923_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_generation_fuel_by_generator_energy_source_monthly_eia923_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'prime_mover_code', 'energy_source_code', name=op.f('pk_generation_fuel_by_generator_energy_source_monthly_eia923')) + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('energy_disposition_type', sa.Text(), nullable=False, comment='Type of energy disposition reported in the core_ferc1__yearly_energy_dispositions_sched401. Dispositions include sales to customers, re-sales of energy, energy used internally, losses, etc.'), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_energy_dispositions_sched401_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_energy_dispositions_sched401_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_disposition_type', name=op.f('pk_out_ferc1__yearly_energy_dispositions_sched401')) ) - op.create_table('generation_fuel_by_generator_monthly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + op.create_table('out_ferc1__yearly_energy_sources_sched401', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_generation_fuel_by_generator_monthly_eia923_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_generation_fuel_by_generator_monthly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_generation_fuel_by_generator_monthly_eia923_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_generation_fuel_by_generator_monthly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_generation_fuel_by_generator_monthly_eia923')) - ) - op.create_table('heat_rate_by_generator_monthly', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), - sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_heat_rate_by_generator_monthly_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_heat_rate_by_generator_monthly_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_heat_rate_by_generator_monthly')) + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('energy_source_type', sa.Text(), nullable=False, comment='Type of energy source reported in the core_ferc1__yearly_energy_sources_sched401 table. There are three groups of energy sources: generation, power exchanges and transmission.'), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.Column('energy_mwh', sa.Float(), nullable=True, comment='Sources and uses of energy in MWh.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_energy_sources_sched401_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_energy_sources_sched401_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'energy_source_type', name=op.f('pk_out_ferc1__yearly_energy_sources_sched401')) ) - op.create_table('income_statement_ferc1', + op.create_table('out_ferc1__yearly_income_statements_sched114', sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), sa.Column('income_type', sa.Text(), nullable=False, comment='Type of income reported in income_statement_ferc1 table.'), @@ -1840,62 +2014,46 @@ def upgrade() -> None: sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_income_statement_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'income_type', name=op.f('pk_income_statement_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_income_statements_sched114_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_income_statements_sched114_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'income_type', name=op.f('pk_out_ferc1__yearly_income_statements_sched114')) ) - op.create_table('mcoe_generators_monthly', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + op.create_table('out_ferc1__yearly_operating_expenses_sched320', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), - sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), - sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), - sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), - sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), - sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), - sa.ForeignKeyConstraint(['energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_mcoe_generators_monthly_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_mcoe_generators_monthly_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_mcoe_generators_monthly_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_mcoe_generators_monthly_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia'], ['utilities_entity_eia.utility_id_eia'], name=op.f('fk_mcoe_generators_monthly_utility_id_eia_utilities_entity_eia')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_mcoe_generators_monthly_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_mcoe_generators_monthly')) - ) - op.create_table('mcoe_monthly', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), - sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_mcoe_monthly_plant_id_eia_generators_entity_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_mcoe_monthly')) + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), + sa.Column('expense_type', sa.Text(), nullable=False, comment='The type of expense.'), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_operating_expenses_sched320_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_operating_expenses_sched320_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'expense_type', name=op.f('pk_out_ferc1__yearly_operating_expenses_sched320')) + ) + op.create_table('out_ferc1__yearly_operating_revenues_sched300', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('revenue_type', sa.Text(), nullable=False, comment='Label describing types of revenues.'), + sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), + sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), + sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), + sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_operating_revenues_sched300_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_operating_revenues_sched300_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'revenue_type', name=op.f('pk_out_ferc1__yearly_operating_revenues_sched300')) ) - op.create_table('other_regulatory_liabilities_ferc1', + op.create_table('out_ferc1__yearly_other_regulatory_liabilities_sched278', sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('description', sa.Text(), nullable=True, comment='Long human-readable description of the meaning of a code/label.'), sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), @@ -1903,124 +2061,59 @@ def upgrade() -> None: sa.Column('increase_in_other_regulatory_liabilities', sa.Float(), nullable=True, comment='The increase during the reporting period of other regulatory liabilities.'), sa.Column('account_detail', sa.Text(), nullable=True, comment='Description of the account number credited from making debit adjustment to other regulatory liabilities.'), sa.Column('decrease_in_other_regulatory_liabilities', sa.Float(), nullable=True, comment='The decrease during the reporting period of other regulatory liabilities.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_other_regulatory_liabilities_ferc1_utility_id_ferc1_utilities_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_other_regulatory_liabilities_sched278_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_other_regulatory_liabilities_sched278_utility_id_pudl_core_pudl__entity_utilities_pudl')) ) - op.create_table('ownership_eia860', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('owner_utility_id_eia', sa.Integer(), nullable=False, comment='The EIA utility Identification number for the owner company that is responsible for the day-to-day operations of the generator, not the operator utility.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('owner_utility_name_eia', sa.Text(), nullable=True, comment='The name of the EIA owner utility.'), - sa.Column('owner_state', sa.Enum('OH', 'NT', 'ID', 'ND', 'MP', 'NM', 'PA', 'IA', 'MD', 'WV', 'AK', 'CT', 'SC', 'TX', 'CA', 'TN', 'UT', 'KY', 'GU', 'MO', 'SD', 'WA', 'NE', 'ON', 'NL', 'KS', 'MA', 'RI', 'WY', 'FL', 'IN', 'MS', 'YT', 'NH', 'OK', 'IL', 'VT', 'SK', 'ME', 'GA', 'PE', 'AS', 'MB', 'AL', 'NC', 'NJ', 'NS', 'PR', 'AR', 'MN', 'OR', 'NV', 'AB', 'MT', 'NB', 'HI', 'DC', 'NY', 'BC', 'CO', 'LA', 'MI', 'NU', 'WI', 'AZ', 'QC', 'VI', 'VA', 'DE'), nullable=True, comment='Two letter ISO-3166 political subdivision code.'), - sa.Column('owner_city', sa.Text(), nullable=True, comment='City of owner.'), - sa.Column('owner_country', sa.Enum('USA', 'CAN'), nullable=True, comment='Three letter ISO-3166 country code.'), - sa.Column('owner_street_address', sa.Text(), nullable=True, comment='Steet address of owner.'), - sa.Column('owner_zip_code', sa.Text(), nullable=True, comment='Zip code of owner.'), - sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_ownership_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_ownership_eia860_plant_id_eia_generators_entity_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'owner_utility_id_eia', name=op.f('pk_ownership_eia860')) - ) - op.create_table('plant_in_service_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + op.create_table('out_ferc1__yearly_plant_in_service_sched204', sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('ferc_account_label', sa.Text(), nullable=False, comment='Long FERC account identifier derived from values reported in the XBRL taxonomies. May also refer to aggregations of individual FERC accounts.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('plant_status', sa.Text(), nullable=True, comment='Utility plant financial status (in service, future, leased, total).'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('additions', sa.Float(), nullable=True, comment='Cost of acquisition of items classified within the account.'), + sa.Column('adjustments', sa.Float(), nullable=True, comment='Cost of adjustments to the account.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), + sa.Column('ferc_account_label', sa.Text(), nullable=False, comment='Long FERC account identifier derived from values reported in the XBRL taxonomies. May also refer to aggregations of individual FERC accounts.'), + sa.Column('retirements', sa.Float(), nullable=True, comment='Cost of disposal of items classified within the account.'), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), sa.Column('starting_balance', sa.Float(), nullable=True, comment='Account balance at beginning of year.'), - sa.Column('additions', sa.Float(), nullable=True, comment='Cost of acquisition of items classified within the account.'), - sa.Column('retirements', sa.Float(), nullable=True, comment='Cost of disposal of items classified within the account.'), - sa.Column('adjustments', sa.Float(), nullable=True, comment='Cost of adjustments to the account.'), sa.Column('transfers', sa.Float(), nullable=True, comment='Cost of transfers into (out of) the account.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_plant_in_service_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'ferc_account_label', name=op.f('pk_plant_in_service_ferc1')) - ) - op.create_table('plants_eia860', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('ash_impoundment', sa.Boolean(), nullable=True, comment='Is there an ash impoundment (e.g. pond, reservoir) at the plant?'), - sa.Column('ash_impoundment_lined', sa.Boolean(), nullable=True, comment='If there is an ash impoundment at the plant, is the impoundment lined?'), - sa.Column('ash_impoundment_status', sa.Text(), nullable=True, comment='If there is an ash impoundment at the plant, the ash impoundment status as of December 31 of the reporting year.'), - sa.Column('balancing_authority_code_eia', sa.Text(), nullable=True, comment='EIA short code identifying a balancing authority.'), - sa.Column('balancing_authority_name_eia', sa.Text(), nullable=True, comment='Name of the balancing authority.'), - sa.Column('datum', sa.Text(), nullable=True, comment='Geodetic coordinate system identifier (e.g. NAD27, NAD83, or WGS84).'), - sa.Column('energy_storage', sa.Boolean(), nullable=True, comment='Indicates if the facility has energy storage capabilities.'), - sa.Column('ferc_cogen_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC cogenerator status. See FERC Form 556.'), - sa.Column('ferc_cogen_status', sa.Boolean(), nullable=True, comment='Indicates whether the plant has FERC qualifying facility cogenerator status. See FERC Form 556.'), - sa.Column('ferc_exempt_wholesale_generator_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC qualifying facility exempt wholesale generator status.'), - sa.Column('ferc_exempt_wholesale_generator', sa.Boolean(), nullable=True, comment='Indicates whether the plant has FERC qualifying facility exempt wholesale generator status'), - sa.Column('ferc_small_power_producer_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC qualifying facility small power producer status. See FERC Form 556.'), - sa.Column('ferc_small_power_producer', sa.Boolean(), nullable=True, comment='Indicates whether the plant has FERC qualifying facility small power producer status. See FERC Form 556.'), - sa.Column('ferc_qualifying_facility_docket_no', sa.Text(), nullable=True, comment='The docket number relating to the FERC qualifying facility cogenerator status. See FERC Form 556.'), - sa.Column('grid_voltage_1_kv', sa.Float(), nullable=True, comment="Plant's grid voltage at point of interconnection to transmission or distibution facilities"), - sa.Column('grid_voltage_2_kv', sa.Float(), nullable=True, comment="Plant's grid voltage at point of interconnection to transmission or distibution facilities"), - sa.Column('grid_voltage_3_kv', sa.Float(), nullable=True, comment="Plant's grid voltage at point of interconnection to transmission or distibution facilities"), - sa.Column('iso_rto_code', sa.Text(), nullable=True, comment="The code of the plant's ISO or RTO. NA if not reported in that year."), - sa.Column('liquefied_natural_gas_storage', sa.Boolean(), nullable=True, comment='Indicates if the facility have the capability to store the natural gas in the form of liquefied natural gas.'), - sa.Column('natural_gas_local_distribution_company', sa.Text(), nullable=True, comment='Names of Local Distribution Company (LDC), connected to natural gas burning power plants.'), - sa.Column('natural_gas_storage', sa.Boolean(), nullable=True, comment='Indicates if the facility have on-site storage of natural gas.'), - sa.Column('natural_gas_pipeline_name_1', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'), - sa.Column('natural_gas_pipeline_name_2', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'), - sa.Column('natural_gas_pipeline_name_3', sa.Text(), nullable=True, comment='The name of the owner or operator of natural gas pipeline that connects directly to this facility or that connects to a lateral pipeline owned by this facility.'), - sa.Column('nerc_region', sa.Enum('BASN', 'CALN', 'CALS', 'DSW', 'ASCC', 'ISONE', 'ERCOT', 'NORW', 'NYISO', 'PJM', 'ROCK', 'ECAR', 'FRCC', 'HICC', 'MAAC', 'MAIN', 'MAPP', 'MRO', 'NPCC', 'RFC', 'SERC', 'SPP', 'TRE', 'WECC', 'WSCC', 'MISO', 'ECAR_MAAC', 'MAPP_WECC', 'RFC_SERC', 'SPP_WECC', 'MRO_WECC', 'ERCOT_SPP', 'SPP_TRE', 'ERCOT_TRE', 'MISO_TRE', 'VI', 'GU', 'PR', 'AS', 'UNK'), nullable=True, comment='NERC region in which the plant is located'), - sa.Column('net_metering', sa.Boolean(), nullable=True, comment='Did this plant have a net metering agreement in effect during the reporting year? (Only displayed for facilities that report the sun or wind as an energy source). This field was only reported up until 2015'), - sa.Column('pipeline_notes', sa.Text(), nullable=True, comment='Additional owner or operator of natural gas pipeline.'), - sa.Column('primary_purpose_id_naics', sa.Integer(), nullable=True, comment='North American Industry Classification System (NAICS) code that best describes the primary purpose of the reporting plant'), - sa.Column('regulatory_status_code', sa.Text(), nullable=True, comment='Indicates whether the plant is regulated or non-regulated.'), - sa.Column('reporting_frequency_code', sa.Enum('A', 'AM', 'M'), nullable=True, comment='Code that specifies what time period data has to be reported (i.e. monthly data or annual totals) and how often the power plant reports this data to EIA. See reporting_frequencies_eia for more details.'), - sa.Column('sector_id_eia', sa.Integer(), nullable=True, comment='EIA assigned sector ID, corresponding to high level NAICS sector, designated by the primary purpose, regulatory status and plant-level combined heat and power status'), - sa.Column('sector_name_eia', sa.Text(), nullable=True, comment='EIA assigned sector name, corresponding to high level NAICS sector, designated by the primary purpose, regulatory status and plant-level combined heat and power status'), - sa.Column('service_area', sa.Text(), nullable=True, comment="Service area in which plant is located; for unregulated companies, it's the electric utility with which plant is interconnected"), - sa.Column('transmission_distribution_owner_id', sa.Integer(), nullable=True, comment='EIA-assigned code for owner of transmission/distribution system to which the plant is interconnected.'), - sa.Column('transmission_distribution_owner_name', sa.Text(), nullable=True, comment='Name of the owner of the transmission or distribution system to which the plant is interconnected.'), - sa.Column('transmission_distribution_owner_state', sa.Text(), nullable=True, comment='State location for owner of transmission/distribution system to which the plant is interconnected.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('water_source', sa.Text(), nullable=True, comment='Name of water source associated with the plant.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['balancing_authority_code_eia'], ['balancing_authorities_eia.code'], name=op.f('fk_plants_eia860_balancing_authority_code_eia_balancing_authorities_eia')), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_plants_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia'], ['plants_entity_eia.plant_id_eia'], name=op.f('fk_plants_eia860_plant_id_eia_plants_entity_eia')), - sa.ForeignKeyConstraint(['reporting_frequency_code'], ['reporting_frequencies_eia.code'], name=op.f('fk_plants_eia860_reporting_frequency_code_reporting_frequencies_eia')), - sa.ForeignKeyConstraint(['sector_id_eia'], ['sector_consolidated_eia.code'], name=op.f('fk_plants_eia860_sector_id_eia_sector_consolidated_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_plants_eia860_utility_id_eia_utilities_eia860')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', name=op.f('pk_plants_eia860')) - ) - op.create_table('plants_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=False, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_plants_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_plants_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'plant_name_ferc1', name=op.f('pk_plants_ferc1')) + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_plant_in_service_sched204_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_plant_in_service_sched204_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'ferc_account_label', name=op.f('pk_out_ferc1__yearly_plant_in_service_sched204')) ) - op.create_table('purchased_power_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + op.create_table('out_ferc1__yearly_purchased_power_and_exchanges_sched326', sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('seller_name', sa.Text(), nullable=True, comment='Name of the seller, or the other party in an exchange transaction.'), - sa.Column('purchase_type_code', sa.Text(), nullable=True, comment="Categorization based on the original contractual terms and conditions of the service. Must be one of 'requirements', 'long_firm', 'intermediate_firm', 'short_firm', 'long_unit', 'intermediate_unit', 'electricity_exchange', 'other_service', or 'adjustment'. Requirements service is ongoing high reliability service, with load integrated into system resource planning. 'Long term' means 5+ years. 'Intermediate term' is 1-5 years. 'Short term' is less than 1 year. 'Firm' means not interruptible for economic reasons. 'unit' indicates service from a particular designated generating unit. 'exchange' is an in-kind transaction."), - sa.Column('tariff', sa.Text(), nullable=True, comment='FERC Rate Schedule Number or Tariff. (Note: may be incomplete if originally reported on multiple lines.)'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('billing_demand_mw', sa.Float(), nullable=True, comment='Monthly average billing demand (for requirements purchases, and any transactions involving demand charges). In megawatts.'), - sa.Column('non_coincident_peak_demand_mw', sa.Float(), nullable=True, comment='Average monthly non-coincident peak (NCP) demand (for requirements purhcases, and any transactions involving demand charges). Monthly NCP demand is the maximum metered hourly (60-minute integration) demand in a month. In megawatts.'), sa.Column('coincident_peak_demand_mw', sa.Float(), nullable=True, comment="Average monthly coincident peak (CP) demand (for requirements purchases, and any transactions involving demand charges). Monthly CP demand is the metered demand during the hour (60-minute integration) in which the supplier's system reaches its monthly peak. In megawatts."), - sa.Column('purchased_mwh', sa.Float(), nullable=True, comment='Megawatt-hours shown on bills rendered to the respondent. Includes both electricity purchased for storage and non-storage purposes, which were lumped together prior to 2021.'), - sa.Column('purchased_storage_mwh', sa.Float(), nullable=True, comment='Number of megawatt hours purchased during the period for energy storage.'), - sa.Column('purchased_other_than_storage_mwh', sa.Float(), nullable=True, comment='Number of megawatt hours purchased during the period for other than energy storage.'), - sa.Column('received_mwh', sa.Float(), nullable=True, comment='Gross megawatt-hours received in power exchanges and used as the basis for settlement.'), sa.Column('delivered_mwh', sa.Float(), nullable=True, comment='Gross megawatt-hours delivered in power exchanges and used as the basis for settlement.'), sa.Column('demand_charges', sa.Float(), nullable=True, comment='Demand charges (USD).'), sa.Column('energy_charges', sa.Float(), nullable=True, comment='Energy charges (USD).'), + sa.Column('non_coincident_peak_demand_mw', sa.Float(), nullable=True, comment='Average monthly non-coincident peak (NCP) demand (for requirements purhcases, and any transactions involving demand charges). Monthly NCP demand is the maximum metered hourly (60-minute integration) demand in a month. In megawatts.'), sa.Column('other_charges', sa.Float(), nullable=True, comment='Other charges, including out-of-period adjustments (USD).'), + sa.Column('purchase_type_code', sa.Text(), nullable=True, comment="Categorization based on the original contractual terms and conditions of the service. Must be one of 'requirements', 'long_firm', 'intermediate_firm', 'short_firm', 'long_unit', 'intermediate_unit', 'electricity_exchange', 'other_service', or 'adjustment'. Requirements service is ongoing high reliability service, with load integrated into system resource planning. 'Long term' means 5+ years. 'Intermediate term' is 1-5 years. 'Short term' is less than 1 year. 'Firm' means not interruptible for economic reasons. 'unit' indicates service from a particular designated generating unit. 'exchange' is an in-kind transaction."), + sa.Column('purchased_mwh', sa.Float(), nullable=True, comment='Megawatt-hours shown on bills rendered to the respondent. Includes both electricity purchased for storage and non-storage purposes, which were lumped together prior to 2021.'), + sa.Column('purchased_storage_mwh', sa.Float(), nullable=True, comment='Number of megawatt hours purchased during the period for energy storage.'), + sa.Column('purchased_other_than_storage_mwh', sa.Float(), nullable=True, comment='Number of megawatt hours purchased during the period for other than energy storage.'), + sa.Column('received_mwh', sa.Float(), nullable=True, comment='Gross megawatt-hours received in power exchanges and used as the basis for settlement.'), + sa.Column('tariff', sa.Text(), nullable=True, comment='FERC Rate Schedule Number or Tariff. (Note: may be incomplete if originally reported on multiple lines.)'), sa.Column('total_settlement', sa.Float(), nullable=True, comment='Sum of demand, energy, and other charges (USD). For power exchanges, the settlement amount for the net receipt of energy. If more energy was delivered than received, this amount is negative.'), - sa.ForeignKeyConstraint(['purchase_type_code'], ['power_purchase_types_ferc1.code'], name=op.f('fk_purchased_power_ferc1_purchase_type_code_power_purchase_types_ferc1')), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_purchased_power_ferc1_utility_id_ferc1_utilities_ferc1')) + sa.ForeignKeyConstraint(['purchase_type_code'], ['core_ferc1__codes_power_purchase_types.code'], name=op.f('fk_out_ferc1__yearly_purchased_power_and_exchanges_sched326_purchase_type_code_core_ferc1__codes_power_purchase_types')), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_purchased_power_and_exchanges_sched326_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_purchased_power_and_exchanges_sched326_utility_id_pudl_core_pudl__entity_utilities_pudl')) ) - op.create_table('retained_earnings_ferc1', + op.create_table('out_ferc1__yearly_retained_earnings_sched118', sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('earnings_type', sa.Text(), nullable=False, comment='Label describing types of earnings.'), @@ -2029,12 +2122,33 @@ def upgrade() -> None: sa.Column('balance', sa.Text(), nullable=True, comment='Indication of whether a column is a credit or debit, as reported in the XBRL taxonomy.'), sa.Column('ferc_account', sa.Text(), nullable=True, comment="Actual FERC Account number (e.g. '359.1') if available, or a PUDL assigned ID when FERC accounts have been split or combined in reporting."), sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_retained_earnings_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'earnings_type', name=op.f('pk_retained_earnings_ferc1')) + sa.Column('utility_type', sa.Text(), nullable=True, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_retained_earnings_sched118_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_retained_earnings_sched118_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'earnings_type', name=op.f('pk_out_ferc1__yearly_retained_earnings_sched118')) + ) + op.create_table('out_ferc1__yearly_sales_by_rate_schedules_sched304', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('rate_schedule_type', sa.Text(), nullable=True, comment='Categorization of rate schedule type.'), + sa.Column('billing_status', sa.Text(), nullable=True, comment='Whether an amount is billed, unbilled, or both. Billed amounts pertain to the exchange of energy and unbilled amounts pertain to other sources of revenue such as contracts with peaker plants to keep them on standby or charging rent to host cell antennas on transmission towers.'), + sa.Column('rate_schedule_description', sa.Text(), nullable=True, comment='Free-form description of what the rate schedule name is. Not standardized. Often a sub-category of rate_schedule_type.'), + sa.Column('sales_mwh', sa.Float(), nullable=True, comment='Quantity of electricity sold in MWh.'), + sa.Column('dollar_value', sa.Float(), nullable=True, comment='Dollar value of reported income, expense, asset, or liability.'), + sa.Column('avg_customers_per_month', sa.Float(), nullable=True, comment='Average number of customers per month.'), + sa.Column('kwh_per_customer', sa.Float(), nullable=True, comment='kwh per customer.'), + sa.Column('revenue_per_kwh', sa.Float(), nullable=True), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_sales_by_rate_schedules_sched304_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_sales_by_rate_schedules_sched304_utility_id_pudl_core_pudl__entity_utilities_pudl')) ) - op.create_table('transmission_statistics_ferc1', + op.create_table('out_ferc1__yearly_transmission_lines_sched422', sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('start_point', sa.Text(), nullable=True, comment='The starting point of a transmission line.'), sa.Column('end_point', sa.Text(), nullable=True, comment='The end point of a transmission line.'), @@ -2045,40 +2159,258 @@ def upgrade() -> None: sa.Column('transmission_line_and_structures_length_miles', sa.Float(), nullable=True, comment='Length (in pole miles or circuit miles (if transmission lines are underground)) for lines that are agrregated with other lines / structures (whose cost are aggregated and combined with other structures).'), sa.Column('num_transmission_circuits', sa.Integer(), nullable=True, comment='Number of circuits in a transmission line.'), sa.Column('conductor_size_and_material', sa.Text(), nullable=True, comment='Size of transmission conductor and material of the transmission line.'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of Land and land rights for the transmission line.'), - sa.Column('capex_other', sa.Float(), nullable=True, comment='Construction and other costs for the transmission line.'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total costs for the transmission line.'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Operating expenses for the transmission line.'), - sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Maintenance expenses for the transmission line.'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Rent expenses for the transmission line.'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Overall expenses for the transmission line.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_transmission_statistics_ferc1_utility_id_ferc1_utilities_ferc1')) + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_other', sa.Float(), nullable=True, comment='Other costs associated with the plant (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_transmission_lines_sched422_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_transmission_lines_sched422_utility_id_pudl_core_pudl__entity_utilities_pudl')) + ) + op.create_table('out_ferc1__yearly_utility_plant_summary_sched200', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), + sa.Column('utility_type_other', sa.Text(), nullable=True, comment='Freeform description of type of utility reported in one of the other three other utility_type sections in the core_ferc1__yearly_utility_plant_summary_sched200 table. This field is reported only in the DBF reporting years (1994-2020).'), + sa.Column('utility_plant_asset_type', sa.Text(), nullable=False, comment='Type of utility plant asset reported in the core_ferc1__yearly_utility_plant_summary_sched200 table. Assets include those leased to others, held for future use, construction work-in-progress and details of accumulated depreciation.'), + sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), + sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['utility_id_ferc1'], ['core_pudl__assn_ferc1_pudl_utilities.utility_id_ferc1'], name=op.f('fk_out_ferc1__yearly_utility_plant_summary_sched200_utility_id_ferc1_core_pudl__assn_ferc1_pudl_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_utility_plant_summary_sched200_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'utility_plant_asset_type', name=op.f('pk_out_ferc1__yearly_utility_plant_summary_sched200')) + ) + op.create_table('_out_eia__plants_utilities', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk__out_eia__plants_utilities_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['core_eia860__scd_plants.plant_id_eia', 'core_eia860__scd_plants.report_date'], name=op.f('fk__out_eia__plants_utilities_plant_id_eia_core_eia860__scd_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_eia__plants_utilities_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk__out_eia__plants_utilities_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_eia__plants_utilities_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'utility_id_eia', name=op.f('pk__out_eia__plants_utilities')) + ) + op.create_table('_out_eia__yearly_heat_rate_by_unit', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=False, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['core_eia860__scd_plants.plant_id_eia', 'core_eia860__scd_plants.report_date'], name=op.f('fk__out_eia__yearly_heat_rate_by_unit_plant_id_eia_core_eia860__scd_plants')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'unit_id_pudl', name=op.f('pk__out_eia__yearly_heat_rate_by_unit')) + ) + op.create_table('_out_ferc1__yearly_hydroelectric_plants_sched406', + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), + sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), + sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), + sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('net_capacity_adverse_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the least favorable operating conditions, in megawatts.'), + sa.Column('net_capacity_favorable_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the most favorable operating conditions, in megawatts.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), + sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), + sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), + sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), + sa.Column('opex_hydraulic', sa.Float(), nullable=True, comment='Production expenses: hydraulic expenses (USD).'), + sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), + sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_type', sa.Text(), nullable=True), + sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_ferc1__yearly_hydroelectric_plants_sched406_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk__out_ferc1__yearly_hydroelectric_plants_sched406_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_ferc1__yearly_hydroelectric_plants_sched406_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id', name=op.f('pk__out_ferc1__yearly_hydroelectric_plants_sched406')) + ) + op.create_table('_out_ferc1__yearly_plants_utilities', + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=False, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_ferc1__yearly_plants_utilities_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk__out_ferc1__yearly_plants_utilities_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_ferc1__yearly_plants_utilities_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('utility_id_ferc1', 'plant_name_ferc1', name=op.f('pk__out_ferc1__yearly_plants_utilities')) + ) + op.create_table('_out_ferc1__yearly_pumped_storage_plants_sched408', + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), + sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('capex_equipment_electric', sa.Float(), nullable=True, comment='Cost of plant: accessory electric equipment (USD).'), + sa.Column('capex_equipment_misc', sa.Float(), nullable=True, comment='Cost of plant: miscellaneous power plant equipment (USD).'), + sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), + sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('capex_wheels_turbines_generators', sa.Float(), nullable=True, comment='Cost of plant: water wheels, turbines, and generators (USD).'), + sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('energy_used_for_pumping_mwh', sa.Float(), nullable=True, comment='Energy used for pumping, in megawatt-hours.'), + sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('net_load_mwh', sa.Float(), nullable=True, comment='Net output for load (net generation - energy used for pumping) in megawatt-hours.'), + sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), + sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), + sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), + sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), + sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), + sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), + sa.Column('opex_production_before_pumping', sa.Float(), nullable=True, comment='Total production expenses before pumping (USD).'), + sa.Column('opex_pumped_storage', sa.Float(), nullable=True, comment='Production expenses: pumped storage (USD).'), + sa.Column('opex_pumping', sa.Float(), nullable=True, comment='Production expenses: We are here to PUMP YOU UP! (USD).'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), + sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_ferc1__yearly_pumped_storage_plants_sched408_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk__out_ferc1__yearly_pumped_storage_plants_sched408_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_ferc1__yearly_pumped_storage_plants_sched408_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id', name=op.f('pk__out_ferc1__yearly_pumped_storage_plants_sched408')) ) - op.create_table('utilities_ferc1_dbf', + op.create_table('_out_ferc1__yearly_small_plants_sched410', + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_ferc1_dbf', sa.Integer(), nullable=False, comment='FERC-assigned respondent_id from DBF reporting years, identifying the reporting entity. Stable from year to year.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_utilities_ferc1_dbf_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1_dbf', name=op.f('pk_utilities_ferc1_dbf')) + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_type', sa.Text(), nullable=True), + sa.Column('license_id_ferc1', sa.Integer(), nullable=True, comment='FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), + sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_type', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_ferc1__yearly_small_plants_sched410_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk__out_ferc1__yearly_small_plants_sched410_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_ferc1__yearly_small_plants_sched410_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id', name=op.f('pk__out_ferc1__yearly_small_plants_sched410')) ) - op.create_table('utilities_ferc1_xbrl', + op.create_table('_out_ferc1__yearly_steam_plants_sched402', + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_ferc1_xbrl', sa.Text(), nullable=False, comment='FERC-assigned entity_id from XBRL reporting years, identifying the reporting entity. Stable from year to year.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_utilities_ferc1_xbrl_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1_xbrl', name=op.f('pk_utilities_ferc1_xbrl')) - ) - op.create_table('utility_plant_summary_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_type', sa.Text(), nullable=False, comment='Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.'), - sa.Column('utility_type_other', sa.Text(), nullable=True, comment='Freeform description of type of utility reported in one of the other three other utility_type sections in the utility_plant_summary_ferc1 table. This field is reported only in the DBF reporting years (1994-2020).'), - sa.Column('utility_plant_asset_type', sa.Text(), nullable=False, comment='Type of utility plant asset reported in the utility_plant_summary_ferc1 table. Assets include those leased to others, held for future use, construction work-in-progress and details of accumulated depreciation.'), - sa.Column('row_type_xbrl', sa.Enum('calculated_value', 'reported_value', 'correction'), nullable=True, comment='Indicates whether the value reported in the row is calculated, or uniquely reported within the table.'), - sa.Column('ending_balance', sa.Float(), nullable=True, comment='Account balance at end of year.'), - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['utility_id_ferc1'], ['utilities_ferc1.utility_id_ferc1'], name=op.f('fk_utility_plant_summary_ferc1_utility_id_ferc1_utilities_ferc1')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'report_year', 'utility_type', 'utility_plant_asset_type', name=op.f('pk_utility_plant_summary_ferc1')) + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_id_ferc1', sa.Integer(), nullable=True, comment='Algorithmically assigned PUDL FERC Plant ID. WARNING: NOT STABLE BETWEEN PUDL DB INITIALIZATIONS.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), + sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('capex_annual_addition', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total`.'), + sa.Column('capex_annual_addition_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total`.'), + sa.Column('capex_annual_per_kw', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per kw.'), + sa.Column('capex_annual_per_mw', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per MW.'), + sa.Column('capex_annual_per_mw_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total` per MW.'), + sa.Column('capex_annual_per_mwh', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per MWh.'), + sa.Column('capex_annual_per_mwh_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total` per MWh.'), + sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('capex_wo_retirement_total', sa.Float(), nullable=True, comment='Total cost of plant (USD) without retirements.'), + sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('not_water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when not limited by condenser water.'), + sa.Column('opex_allowances', sa.Float(), nullable=True, comment='Allowances.'), + sa.Column('opex_boiler', sa.Float(), nullable=True, comment='Maintenance of boiler (or reactor) plant.'), + sa.Column('opex_coolants', sa.Float(), nullable=True, comment='Cost of coolants and water (nuclear plants only)'), + sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), + sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), + sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), + sa.Column('opex_fuel_per_mwh', sa.Float(), nullable=True, comment='Production expenses: fuel (USD) per megawatt-hour (Mwh).'), + sa.Column('opex_misc_power', sa.Float(), nullable=True, comment='Miscellaneous steam (or nuclear) expenses.'), + sa.Column('opex_misc_steam', sa.Float(), nullable=True, comment='Maintenance of miscellaneous steam (or nuclear) plant.'), + sa.Column('opex_nonfuel_per_mwh', sa.Float(), nullable=True, comment='Investments in non-fuel production expenses per Mwh.'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), + sa.Column('opex_plants', sa.Float(), nullable=True, comment='Maintenance of electrical plant.'), + sa.Column('opex_production_total', sa.Float(), nullable=True, comment='Total operating expenses.'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_steam', sa.Float(), nullable=True, comment='Steam expenses.'), + sa.Column('opex_steam_other', sa.Float(), nullable=True, comment='Steam from other sources.'), + sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_transfer', sa.Float(), nullable=True, comment='Steam transferred (Credit).'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), + sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), + sa.Column('plant_type', sa.Text(), nullable=True), + sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when limited by condenser water.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_ferc1__yearly_steam_plants_sched402_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk__out_ferc1__yearly_steam_plants_sched402_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_ferc1__yearly_steam_plants_sched402_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id', name=op.f('pk__out_ferc1__yearly_steam_plants_sched402')) ) - op.create_table('boilers_eia860', + op.create_table('core_eia860__scd_boilers', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), @@ -2170,109 +2502,318 @@ def upgrade() -> None: sa.Column('so2_control_proposed_strategy_3', sa.Text(), nullable=True, comment='Proposed strategy to comply with the most stringent sulfur dioxide regulation.'), sa.Column('standard_so2_percent_scrubbed', sa.Float(), nullable=True, comment='The percent of sulfur dioxide to be scrubbed specified by the most stringent sulfur dioxide regulation.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['boiler_fuel_code_1'], ['energy_sources_eia.code'], name=op.f('fk_boilers_eia860_boiler_fuel_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_fuel_code_2'], ['energy_sources_eia.code'], name=op.f('fk_boilers_eia860_boiler_fuel_code_2_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_fuel_code_3'], ['energy_sources_eia.code'], name=op.f('fk_boilers_eia860_boiler_fuel_code_3_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_fuel_code_4'], ['energy_sources_eia.code'], name=op.f('fk_boilers_eia860_boiler_fuel_code_4_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_status'], ['boiler_status_eia.code'], name=op.f('fk_boilers_eia860_boiler_status_boiler_status_eia')), - sa.ForeignKeyConstraint(['boiler_type'], ['boiler_types_eia.code'], name=op.f('fk_boilers_eia860_boiler_type_boiler_types_eia')), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_boilers_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['firing_type_1'], ['firing_types_eia.code'], name=op.f('fk_boilers_eia860_firing_type_1_firing_types_eia')), - sa.ForeignKeyConstraint(['firing_type_2'], ['firing_types_eia.code'], name=op.f('fk_boilers_eia860_firing_type_2_firing_types_eia')), - sa.ForeignKeyConstraint(['firing_type_3'], ['firing_types_eia.code'], name=op.f('fk_boilers_eia860_firing_type_3_firing_types_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_1'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_existing_strategy_1_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_2'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_existing_strategy_2_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_3'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_existing_strategy_3_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_4'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_existing_strategy_4_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_5'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_existing_strategy_5_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_6'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_existing_strategy_6_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_1'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_proposed_strategy_1_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_2'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_proposed_strategy_2_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_3'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_mercury_control_proposed_strategy_3_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_1'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_existing_caaa_compliance_strategy_1_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_2'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_existing_caaa_compliance_strategy_2_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_3'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_existing_caaa_compliance_strategy_3_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_manufacturer_code'], ['environmental_equipment_manufacturers_eia.code'], name=op.f('fk_boilers_eia860_nox_control_manufacturer_code_environmental_equipment_manufacturers_eia')), - sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_1'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_out_of_compliance_strategy_1_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_2'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_out_of_compliance_strategy_2_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_3'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_out_of_compliance_strategy_3_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_1'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_planned_caaa_compliance_strategy_1_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_2'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_planned_caaa_compliance_strategy_2_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_3'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_nox_control_planned_caaa_compliance_strategy_3_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_status_code'], ['nox_control_status_eia.code'], name=op.f('fk_boilers_eia860_nox_control_status_code_nox_control_status_eia')), - sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_1'], ['particulate_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_particulate_control_out_of_compliance_strategy_1_particulate_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_2'], ['particulate_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_particulate_control_out_of_compliance_strategy_2_particulate_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_3'], ['particulate_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_particulate_control_out_of_compliance_strategy_3_particulate_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['boilers_entity_eia.plant_id_eia', 'boilers_entity_eia.boiler_id'], name=op.f('fk_boilers_eia860_plant_id_eia_boilers_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['plants_eia860.plant_id_eia', 'plants_eia860.report_date'], name=op.f('fk_boilers_eia860_plant_id_eia_plants_eia860')), - sa.ForeignKeyConstraint(['regulation_mercury'], ['regulations_eia.code'], name=op.f('fk_boilers_eia860_regulation_mercury_regulations_eia')), - sa.ForeignKeyConstraint(['regulation_nox'], ['regulations_eia.code'], name=op.f('fk_boilers_eia860_regulation_nox_regulations_eia')), - sa.ForeignKeyConstraint(['regulation_particulate'], ['regulations_eia.code'], name=op.f('fk_boilers_eia860_regulation_particulate_regulations_eia')), - sa.ForeignKeyConstraint(['regulation_so2'], ['regulations_eia.code'], name=op.f('fk_boilers_eia860_regulation_so2_regulations_eia')), - sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_1'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_existing_caaa_compliance_strategy_1_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_2'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_existing_caaa_compliance_strategy_2_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_3'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_existing_caaa_compliance_strategy_3_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_1'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_out_of_compliance_strategy_1_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_2'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_out_of_compliance_strategy_2_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_3'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_out_of_compliance_strategy_3_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_1'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_planned_caaa_compliance_strategy_1_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_2'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_planned_caaa_compliance_strategy_2_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_3'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_boilers_eia860_so2_control_planned_caaa_compliance_strategy_3_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['unit_nox'], ['nox_units_eia.code'], name=op.f('fk_boilers_eia860_unit_nox_nox_units_eia')), - sa.ForeignKeyConstraint(['unit_particulate'], ['particulate_units_eia.code'], name=op.f('fk_boilers_eia860_unit_particulate_particulate_units_eia')), - sa.ForeignKeyConstraint(['unit_so2'], ['so2_units_eia.code'], name=op.f('fk_boilers_eia860_unit_so2_so2_units_eia')), - sa.ForeignKeyConstraint(['wet_dry_bottom'], ['wet_dry_bottom_eia.code'], name=op.f('fk_boilers_eia860_wet_dry_bottom_wet_dry_bottom_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'report_date', name=op.f('pk_boilers_eia860')) - ) - op.create_table('denorm_fuel_by_plant_ferc1', - sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=False, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('coal_fraction_cost', sa.Float(), nullable=True, comment='Coal cost as a percentage of overall fuel cost.'), - sa.Column('coal_fraction_mmbtu', sa.Float(), nullable=True, comment='Coal heat content as a percentage of overall fuel heat content (mmBTU).'), - sa.Column('fuel_cost', sa.Float(), nullable=True, comment='Total fuel cost for plant (in $USD).'), - sa.Column('fuel_mmbtu', sa.Float(), nullable=True, comment='Total heat content for plant (in MMBtu).'), - sa.Column('gas_fraction_cost', sa.Float(), nullable=True, comment='Natural gas cost as a percentage of overall fuel cost.'), - sa.Column('gas_fraction_mmbtu', sa.Float(), nullable=True, comment='Natural gas heat content as a percentage of overall fuel heat content (MMBtu).'), - sa.Column('nuclear_fraction_cost', sa.Float(), nullable=True, comment='Nuclear cost as a percentage of overall fuel cost.'), - sa.Column('nuclear_fraction_mmbtu', sa.Float(), nullable=True, comment='Nuclear heat content as a percentage of overall fuel heat content (MMBtu).'), - sa.Column('oil_fraction_cost', sa.Float(), nullable=True, comment='Oil cost as a percentage of overall fuel cost.'), - sa.Column('oil_fraction_mmbtu', sa.Float(), nullable=True, comment='Oil heat content as a percentage of overall fuel heat content (MMBtu).'), - sa.Column('primary_fuel_by_cost', sa.Text(), nullable=True, comment='Primary fuel for plant as a percentage of cost.'), - sa.Column('primary_fuel_by_mmbtu', sa.Text(), nullable=True, comment='Primary fuel for plant as a percentage of heat content.'), - sa.Column('waste_fraction_cost', sa.Float(), nullable=True, comment='Waste-heat cost as a percentage of overall fuel cost.'), - sa.Column('waste_fraction_mmbtu', sa.Float(), nullable=True, comment='Waste-heat heat content as a percentage of overall fuel heat content (MMBtu).'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_fuel_by_plant_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_fuel_by_plant_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_fuel_by_plant_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_year', 'utility_id_ferc1', 'plant_name_ferc1', name=op.f('pk_denorm_fuel_by_plant_ferc1')) + sa.ForeignKeyConstraint(['boiler_fuel_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_boilers_boiler_fuel_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_fuel_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_boilers_boiler_fuel_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_fuel_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_boilers_boiler_fuel_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_fuel_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_boilers_boiler_fuel_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_status'], ['core_eia__codes_boiler_status.code'], name=op.f('fk_core_eia860__scd_boilers_boiler_status_core_eia__codes_boiler_status')), + sa.ForeignKeyConstraint(['boiler_type'], ['core_eia__codes_boiler_types.code'], name=op.f('fk_core_eia860__scd_boilers_boiler_type_core_eia__codes_boiler_types')), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__scd_boilers_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['firing_type_1'], ['core_eia__codes_firing_types.code'], name=op.f('fk_core_eia860__scd_boilers_firing_type_1_core_eia__codes_firing_types')), + sa.ForeignKeyConstraint(['firing_type_2'], ['core_eia__codes_firing_types.code'], name=op.f('fk_core_eia860__scd_boilers_firing_type_2_core_eia__codes_firing_types')), + sa.ForeignKeyConstraint(['firing_type_3'], ['core_eia__codes_firing_types.code'], name=op.f('fk_core_eia860__scd_boilers_firing_type_3_core_eia__codes_firing_types')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_1'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_existing_strategy_1_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_2'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_existing_strategy_2_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_3'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_existing_strategy_3_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_4'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_existing_strategy_4_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_5'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_existing_strategy_5_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_6'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_existing_strategy_6_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_1'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_proposed_strategy_1_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_2'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_proposed_strategy_2_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_3'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_mercury_control_proposed_strategy_3_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_1'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_existing_caaa_compliance_strategy_1_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_2'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_existing_caaa_compliance_strategy_2_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_3'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_existing_caaa_compliance_strategy_3_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_manufacturer_code'], ['core_eia__codes_environmental_equipment_manufacturers.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_manufacturer_code_core_eia__codes_environmental_equipment_manufacturers')), + sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_1'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_out_of_compliance_strategy_1_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_2'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_out_of_compliance_strategy_2_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_3'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_out_of_compliance_strategy_3_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_1'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_planned_caaa_compliance_strategy_1_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_2'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_planned_caaa_compliance_strategy_2_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_3'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_planned_caaa_compliance_strategy_3_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_status_code'], ['core_eia__codes_nox_control_status.code'], name=op.f('fk_core_eia860__scd_boilers_nox_control_status_code_core_eia__codes_nox_control_status')), + sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_1'], ['core_eia__codes_particulate_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_particulate_control_out_of_compliance_strategy_1_core_eia__codes_particulate_compliance_strategies')), + sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_2'], ['core_eia__codes_particulate_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_particulate_control_out_of_compliance_strategy_2_core_eia__codes_particulate_compliance_strategies')), + sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_3'], ['core_eia__codes_particulate_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_particulate_control_out_of_compliance_strategy_3_core_eia__codes_particulate_compliance_strategies')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id'], ['core_eia__entity_boilers.plant_id_eia', 'core_eia__entity_boilers.boiler_id'], name=op.f('fk_core_eia860__scd_boilers_plant_id_eia_core_eia__entity_boilers')), + sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['core_eia860__scd_plants.plant_id_eia', 'core_eia860__scd_plants.report_date'], name=op.f('fk_core_eia860__scd_boilers_plant_id_eia_core_eia860__scd_plants')), + sa.ForeignKeyConstraint(['regulation_mercury'], ['core_eia__codes_regulations.code'], name=op.f('fk_core_eia860__scd_boilers_regulation_mercury_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['regulation_nox'], ['core_eia__codes_regulations.code'], name=op.f('fk_core_eia860__scd_boilers_regulation_nox_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['regulation_particulate'], ['core_eia__codes_regulations.code'], name=op.f('fk_core_eia860__scd_boilers_regulation_particulate_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['regulation_so2'], ['core_eia__codes_regulations.code'], name=op.f('fk_core_eia860__scd_boilers_regulation_so2_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_1'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_existing_caaa_compliance_strategy_1_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_2'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_existing_caaa_compliance_strategy_2_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_3'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_existing_caaa_compliance_strategy_3_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_1'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_out_of_compliance_strategy_1_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_2'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_out_of_compliance_strategy_2_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_3'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_out_of_compliance_strategy_3_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_1'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_planned_caaa_compliance_strategy_1_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_2'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_planned_caaa_compliance_strategy_2_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_3'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_core_eia860__scd_boilers_so2_control_planned_caaa_compliance_strategy_3_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['unit_nox'], ['core_eia__codes_nox_units.code'], name=op.f('fk_core_eia860__scd_boilers_unit_nox_core_eia__codes_nox_units')), + sa.ForeignKeyConstraint(['unit_particulate'], ['core_eia__codes_particulate_units.code'], name=op.f('fk_core_eia860__scd_boilers_unit_particulate_core_eia__codes_particulate_units')), + sa.ForeignKeyConstraint(['unit_so2'], ['core_eia__codes_so2_units.code'], name=op.f('fk_core_eia860__scd_boilers_unit_so2_core_eia__codes_so2_units')), + sa.ForeignKeyConstraint(['wet_dry_bottom'], ['core_eia__codes_wet_dry_bottom.code'], name=op.f('fk_core_eia860__scd_boilers_wet_dry_bottom_core_eia__codes_wet_dry_bottom')), + sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'report_date', name=op.f('pk_core_eia860__scd_boilers')) + ) + op.create_table('core_eia860__scd_generators', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), + sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), + sa.Column('ownership_code', sa.Text(), nullable=True, comment='Identifies the ownership for each generator.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('summer_capacity_mw', sa.Float(), nullable=True, comment='The net summer capacity.'), + sa.Column('summer_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the summer capacity value was an estimate'), + sa.Column('winter_capacity_mw', sa.Float(), nullable=True, comment='The net winter capacity.'), + sa.Column('winter_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the winter capacity value was an estimate'), + sa.Column('net_capacity_mwdc', sa.Float(), nullable=True, comment='Generation capacity in megawatts of direct current that is subject to a net metering agreement. Typically used for behind-the-meter solar PV.'), + sa.Column('energy_storage_capacity_mwh', sa.Float(), nullable=True, comment='Energy storage capacity in MWh (e.g. for batteries).'), + sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), + sa.Column('energy_source_code_2', sa.Text(), nullable=True, comment='The code representing the second most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_3', sa.Text(), nullable=True, comment='The code representing the third most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_4', sa.Text(), nullable=True, comment='The code representing the fourth most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_5', sa.Text(), nullable=True, comment='The code representing the fifth most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_6', sa.Text(), nullable=True, comment='The code representing the sixth most predominant type of energy that fuels the generator'), + sa.Column('energy_source_1_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 1.'), + sa.Column('energy_source_1_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 1.'), + sa.Column('energy_source_1_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 1.'), + sa.Column('energy_source_2_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 2.'), + sa.Column('energy_source_2_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 2.'), + sa.Column('energy_source_2_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 2.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('multiple_fuels', sa.Boolean(), nullable=True, comment='Can the generator burn multiple fuels?'), + sa.Column('deliver_power_transgrid', sa.Boolean(), nullable=True, comment='Indicate whether the generator can deliver power to the transmission grid.'), + sa.Column('distributed_generation', sa.Boolean(), nullable=True, comment='Whether the generator is considered distributed generation'), + sa.Column('syncronized_transmission_grid', sa.Boolean(), nullable=True, comment='Indicates whether standby generators (SB status) can be synchronized to the grid.'), + sa.Column('turbines_num', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), + sa.Column('planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are any planned capacity uprates/derates, repowering, other modifications, or generator retirements scheduled for the next 5 years.'), + sa.Column('planned_net_summer_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in summer capacity expected to be realized from the modification to the equipment.'), + sa.Column('planned_net_winter_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in winter capacity expected to be realized from the uprate modification to the equipment.'), + sa.Column('planned_uprate_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the uprate modification.'), + sa.Column('planned_net_summer_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in summer capacity expected to be realized from the derate modification to the equipment.'), + sa.Column('planned_net_winter_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in winter capacity expected to be realized from the derate modification to the equipment.'), + sa.Column('planned_derate_date', sa.Date(), nullable=True, comment='Planned effective month that the generator is scheduled to enter operation after the derate modification.'), + sa.Column('planned_new_prime_mover_code', sa.Text(), nullable=True, comment='New prime mover for the planned repowered generator.'), + sa.Column('planned_energy_source_code_1', sa.Text(), nullable=True, comment='New energy source code for the planned repowered generator.'), + sa.Column('planned_repower_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the repowering is complete.'), + sa.Column('other_planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are there other modifications planned for the generator.'), + sa.Column('other_modifications_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter commercial operation after any other planned modification is complete.'), + sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), + sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), + sa.Column('startup_source_code_1', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_2', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_3', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_4', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), + sa.Column('turbines_inverters_hydrokinetics', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), + sa.Column('time_cold_shutdown_full_load_code', sa.Text(), nullable=True, comment='The minimum amount of time required to bring the unit to full load from shutdown.'), + sa.Column('planned_new_capacity_mw', sa.Float(), nullable=True, comment='The expected new namplate capacity for the generator.'), + sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), + sa.Column('switch_oil_gas', sa.Boolean(), nullable=True, comment='Indicates whether the generator switch between oil and natural gas.'), + sa.Column('nameplate_power_factor', sa.Float(), nullable=True, comment='The nameplate power factor of the generator.'), + sa.Column('minimum_load_mw', sa.Float(), nullable=True, comment='The minimum load at which the generator can operate at continuosuly.'), + sa.Column('uprate_derate_during_year', sa.Boolean(), nullable=True, comment='Was an uprate or derate completed on this generator during the reporting year?'), + sa.Column('uprate_derate_completed_date', sa.Date(), nullable=True, comment='The date when the uprate or derate was completed.'), + sa.Column('current_planned_generator_operating_date', sa.Date(), nullable=True, comment='The most recently updated effective date on which the generator is scheduled to start operation'), + sa.Column('summer_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated summer capacity (in MWh).'), + sa.Column('winter_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated winter capacity (in MWh).'), + sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), + sa.Column('owned_by_non_utility', sa.Boolean(), nullable=True, comment='Whether any part of generator is owned by a nonutilty'), + sa.Column('reactive_power_output_mvar', sa.Float(), nullable=True, comment='Reactive Power Output (MVAr)'), + sa.Column('ferc_qualifying_facility', sa.Boolean(), nullable=True, comment='Indicatates whether or not a generator is a qualifying FERC cogeneation facility.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__scd_generators_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_1_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_1_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_1_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_1_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_2_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_2_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_2_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_5'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_code_5_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_6'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_energy_source_code_6_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['operational_status_code'], ['core_eia__codes_operational_status.code'], name=op.f('fk_core_eia860__scd_generators_operational_status_code_core_eia__codes_operational_status')), + sa.ForeignKeyConstraint(['planned_energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_planned_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['planned_new_prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_core_eia860__scd_generators_planned_new_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['core_eia__entity_generators.plant_id_eia', 'core_eia__entity_generators.generator_id'], name=op.f('fk_core_eia860__scd_generators_plant_id_eia_core_eia__entity_generators')), + sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['core_eia860__scd_plants.plant_id_eia', 'core_eia860__scd_plants.report_date'], name=op.f('fk_core_eia860__scd_generators_plant_id_eia_core_eia860__scd_plants')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_core_eia860__scd_generators_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['startup_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_startup_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_startup_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_startup_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_core_eia860__scd_generators_startup_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_core_eia860__scd_generators_utility_id_eia_core_eia860__scd_utilities')), + sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_core_eia860__scd_generators')) + ) + op.create_table('core_ferc1__yearly_hydroelectric_plants_sched406', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), + sa.Column('plant_type', sa.Enum('run_of_river', 'na_category', 'run_of_river_with_storage', 'storage', 'hydro'), nullable=True), + sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), + sa.Column('net_capacity_favorable_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the most favorable operating conditions, in megawatts.'), + sa.Column('net_capacity_adverse_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the least favorable operating conditions, in megawatts.'), + sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), + sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), + sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), + sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), + sa.Column('opex_hydraulic', sa.Float(), nullable=True, comment='Production expenses: hydraulic expenses (USD).'), + sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), + sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), + sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), + sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), + sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_core_ferc1__yearly_hydroelectric_plants_sched406_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')) + ) + op.create_table('core_ferc1__yearly_pumped_storage_plants_sched408', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), + sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), + sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), + sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('energy_used_for_pumping_mwh', sa.Float(), nullable=True, comment='Energy used for pumping, in megawatt-hours.'), + sa.Column('net_load_mwh', sa.Float(), nullable=True, comment='Net output for load (net generation - energy used for pumping) in megawatt-hours.'), + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), + sa.Column('capex_wheels_turbines_generators', sa.Float(), nullable=True, comment='Cost of plant: water wheels, turbines, and generators (USD).'), + sa.Column('capex_equipment_electric', sa.Float(), nullable=True, comment='Cost of plant: accessory electric equipment (USD).'), + sa.Column('capex_equipment_misc', sa.Float(), nullable=True, comment='Cost of plant: miscellaneous power plant equipment (USD).'), + sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), + sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), + sa.Column('opex_pumped_storage', sa.Float(), nullable=True, comment='Production expenses: pumped storage (USD).'), + sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), + sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), + sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), + sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), + sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), + sa.Column('opex_production_before_pumping', sa.Float(), nullable=True, comment='Total production expenses before pumping (USD).'), + sa.Column('opex_pumping', sa.Float(), nullable=True, comment='Production expenses: We are here to PUMP YOU UP! (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_core_ferc1__yearly_pumped_storage_plants_sched408_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')) ) - op.create_table('denorm_fuel_ferc1', + op.create_table('core_ferc1__yearly_small_plants_sched410', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('plant_type', sa.Text(), nullable=True), + sa.Column('license_id_ferc1', sa.Integer(), nullable=True, comment='FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.'), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), + sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), + sa.Column('fuel_type', sa.Text(), nullable=True), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_core_ferc1__yearly_small_plants_sched410_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')) + ) + op.create_table('core_ferc1__yearly_steam_plants_fuel_sched402', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_total_cost', sa.Float(), nullable=True, comment='Total cost of consumed fuel.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_units', sa.Enum('mmbtu', 'gramsU', 'kg', 'mwhth', 'kgal', 'bbl', 'klbs', 'mcf', 'gal', 'mwdth', 'btu', 'ton'), nullable=True, comment='Reported unit of measure for fuel.'), sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), sa.Column('fuel_cost_per_unit_burned', sa.Float(), nullable=True, comment='Average cost of fuel consumed in the report year per reported fuel unit (USD).'), sa.Column('fuel_cost_per_unit_delivered', sa.Float(), nullable=True, comment='Average cost of fuel delivered in the report year per reported fuel unit (USD).'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_units', sa.Enum('mmbtu', 'gramsU', 'kg', 'mwhth', 'kgal', 'bbl', 'klbs', 'mcf', 'gal', 'mwdth', 'btu', 'ton'), nullable=True, comment='Reported unit of measure for fuel.'), - sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_fuel_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_fuel_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_fuel_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('record_id', name=op.f('pk_denorm_fuel_ferc1')) + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_core_ferc1__yearly_steam_plants_fuel_sched402_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')) + ) + op.create_table('core_ferc1__yearly_steam_plants_sched402', + sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('plant_id_ferc1', sa.Integer(), nullable=True, comment='Algorithmically assigned PUDL FERC Plant ID. WARNING: NOT STABLE BETWEEN PUDL DB INITIALIZATIONS.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('plant_type', sa.Enum('solar_thermal', 'internal_combustion', 'wind', 'combustion_turbine', 'combined_cycle', 'steam', 'na_category', 'photovoltaic', 'nuclear', 'geothermal'), nullable=True), + sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), + sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), + sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), + sa.Column('water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when limited by condenser water.'), + sa.Column('not_water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when not limited by condenser water.'), + sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), + sa.Column('opex_coolants', sa.Float(), nullable=True, comment='Cost of coolants and water (nuclear plants only)'), + sa.Column('opex_steam', sa.Float(), nullable=True, comment='Steam expenses.'), + sa.Column('opex_steam_other', sa.Float(), nullable=True, comment='Steam from other sources.'), + sa.Column('opex_transfer', sa.Float(), nullable=True, comment='Steam transferred (Credit).'), + sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), + sa.Column('opex_misc_power', sa.Float(), nullable=True, comment='Miscellaneous steam (or nuclear) expenses.'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_allowances', sa.Float(), nullable=True, comment='Allowances.'), + sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), + sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('opex_boiler', sa.Float(), nullable=True, comment='Maintenance of boiler (or reactor) plant.'), + sa.Column('opex_plants', sa.Float(), nullable=True, comment='Maintenance of electrical plant.'), + sa.Column('opex_misc_steam', sa.Float(), nullable=True, comment='Maintenance of miscellaneous steam (or nuclear) plant.'), + sa.Column('opex_production_total', sa.Float(), nullable=True, comment='Total operating expenses.'), + sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), + sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_core_ferc1__yearly_steam_plants_sched402_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')) ) - op.create_table('denorm_fuel_receipts_costs_yearly_eia923', + op.create_table('out_eia923__yearly_fuel_receipts_costs', sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), @@ -2293,12 +2834,14 @@ def upgrade() -> None: sa.Column('mercury_content_ppm', sa.Float(), nullable=True, comment='Mercury content in parts per million (ppm) to the nearest 0.001 ppm.'), sa.Column('moisture_content_pct', sa.Float(), nullable=True), sa.Column('chlorine_content_ppm', sa.Float(), nullable=True), - sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['plants_eia860.plant_id_eia', 'plants_eia860.report_date'], name=op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_plant_id_eia_plants_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_pudl_utilities_pudl')) + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__yearly_fuel_receipts_costs_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['core_eia860__scd_plants.plant_id_eia', 'core_eia860__scd_plants.report_date'], name=op.f('fk_out_eia923__yearly_fuel_receipts_costs_plant_id_eia_core_eia860__scd_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__yearly_fuel_receipts_costs_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia923__yearly_fuel_receipts_costs_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__yearly_fuel_receipts_costs_utility_id_pudl_core_pudl__entity_utilities_pudl')) ) - op.create_table('denorm_generation_fuel_combined_yearly_eia923', + op.create_table('out_eia923__yearly_generation_fuel_combined', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), @@ -2315,99 +2858,17 @@ def upgrade() -> None: sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generation_fuel_combined_yearly_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['plants_eia860.plant_id_eia', 'plants_eia860.report_date'], name=op.f('fk_denorm_generation_fuel_combined_yearly_eia923_plant_id_eia_plants_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_generation_fuel_combined_yearly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_generation_fuel_combined_yearly_eia923_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_denorm_generation_fuel_combined_yearly_eia923')) - ) - op.create_table('denorm_plants_all_ferc1', - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_id_ferc1', sa.Integer(), nullable=True, comment='Algorithmically assigned PUDL FERC Plant ID. WARNING: NOT STABLE BETWEEN PUDL DB INITIALIZATIONS.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), - sa.Column('avg_num_employees', sa.Float(), nullable=True), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('capex_annual_addition', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total`.'), - sa.Column('capex_annual_addition_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total`.'), - sa.Column('capex_annual_per_kw', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per kw.'), - sa.Column('capex_annual_per_mw', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per MW.'), - sa.Column('capex_annual_per_mw_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total` per MW.'), - sa.Column('capex_annual_per_mwh', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per MWh.'), - sa.Column('capex_annual_per_mwh_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total` per MWh.'), - sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('capex_wo_retirement_total', sa.Float(), nullable=True, comment='Total cost of plant (USD) without retirements.'), - sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('not_water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when not limited by condenser water.'), - sa.Column('opex_allowances', sa.Float(), nullable=True, comment='Allowances.'), - sa.Column('opex_boiler', sa.Float(), nullable=True, comment='Maintenance of boiler (or reactor) plant.'), - sa.Column('opex_coolants', sa.Float(), nullable=True, comment='Cost of coolants and water (nuclear plants only)'), - sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), - sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), - sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), - sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.Column('opex_misc_power', sa.Float(), nullable=True, comment='Miscellaneous steam (or nuclear) expenses.'), - sa.Column('opex_misc_steam', sa.Float(), nullable=True, comment='Maintenance of miscellaneous steam (or nuclear) plant.'), - sa.Column('opex_nonfuel_per_mwh', sa.Float(), nullable=True, comment='Investments in non-fuel production expenses per Mwh.'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), - sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), - sa.Column('opex_production_total', sa.Float(), nullable=True, comment='Total operating expenses.'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), - sa.Column('opex_steam', sa.Float(), nullable=True, comment='Steam expenses.'), - sa.Column('opex_steam_other', sa.Float(), nullable=True, comment='Steam from other sources.'), - sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), - sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_transfer', sa.Float(), nullable=True, comment='Steam transferred (Credit).'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), - sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), - sa.Column('plant_type', sa.Text(), nullable=True), - sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when limited by condenser water.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('fuel_type', sa.Text(), nullable=True), - sa.Column('license_id_ferc1', sa.Integer(), nullable=True, comment='FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.'), - sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), - sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), - sa.Column('net_capacity_adverse_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the least favorable operating conditions, in megawatts.'), - sa.Column('net_capacity_favorable_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the most favorable operating conditions, in megawatts.'), - sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), - sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), - sa.Column('opex_hydraulic', sa.Float(), nullable=True, comment='Production expenses: hydraulic expenses (USD).'), - sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), - sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), - sa.Column('ferc_license_id', sa.Text(), nullable=True, comment='The FERC license ID of a project.'), - sa.Column('capex_equipment_electric', sa.Float(), nullable=True, comment='Cost of plant: accessory electric equipment (USD).'), - sa.Column('capex_equipment_misc', sa.Float(), nullable=True, comment='Cost of plant: miscellaneous power plant equipment (USD).'), - sa.Column('capex_wheels_turbines_generators', sa.Float(), nullable=True, comment='Cost of plant: water wheels, turbines, and generators (USD).'), - sa.Column('energy_used_for_pumping_mwh', sa.Float(), nullable=True, comment='Energy used for pumping, in megawatt-hours.'), - sa.Column('net_load_mwh', sa.Float(), nullable=True, comment='Net output for load (net generation - energy used for pumping) in megawatt-hours.'), - sa.Column('opex_production_before_pumping', sa.Float(), nullable=True, comment='Total production expenses before pumping (USD).'), - sa.Column('opex_pumped_storage', sa.Float(), nullable=True, comment='Production expenses: pumped storage (USD).'), - sa.Column('opex_pumping', sa.Float(), nullable=True, comment='Production expenses: We are here to PUMP YOU UP! (USD).'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_all_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_plants_all_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_all_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('record_id', name=op.f('pk_denorm_plants_all_ferc1')) - ) - op.create_table('denorm_plants_eia', + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__yearly_generation_fuel_combined_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__yearly_generation_fuel_combined_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['core_eia860__scd_plants.plant_id_eia', 'core_eia860__scd_plants.report_date'], name=op.f('fk_out_eia923__yearly_generation_fuel_combined_plant_id_eia_core_eia860__scd_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__yearly_generation_fuel_combined_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__yearly_generation_fuel_combined_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia923__yearly_generation_fuel_combined_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__yearly_generation_fuel_combined_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'prime_mover_code', 'energy_source_code', name=op.f('pk_out_eia923__yearly_generation_fuel_combined')) + ) + op.create_table('out_eia__yearly_plants', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), sa.Column('city', sa.Text(), nullable=True), @@ -2456,151 +2917,23 @@ def upgrade() -> None: sa.Column('transmission_distribution_owner_name', sa.Text(), nullable=True, comment='Name of the owner of the transmission or distribution system to which the plant is interconnected.'), sa.Column('transmission_distribution_owner_state', sa.Text(), nullable=True, comment='State location for owner of transmission/distribution system to which the plant is interconnected.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('water_source', sa.Text(), nullable=True, comment='Name of water source associated with the plant.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('balancing_authority_code_eia_consistent_rate', sa.Float(), nullable=True, comment='Percentage consistency of balancing authority code across entity records.'), - sa.ForeignKeyConstraint(['balancing_authority_code_eia'], ['balancing_authorities_eia.code'], name=op.f('fk_denorm_plants_eia_balancing_authority_code_eia_balancing_authorities_eia')), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_plants_eia_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['plants_eia860.plant_id_eia', 'plants_eia860.report_date'], name=op.f('fk_denorm_plants_eia_plant_id_eia_plants_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_eia_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['reporting_frequency_code'], ['reporting_frequencies_eia.code'], name=op.f('fk_denorm_plants_eia_reporting_frequency_code_reporting_frequencies_eia')), - sa.ForeignKeyConstraint(['sector_id_eia'], ['sector_consolidated_eia.code'], name=op.f('fk_denorm_plants_eia_sector_id_eia_sector_consolidated_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_plants_eia_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_eia_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', name=op.f('pk_denorm_plants_eia')) - ) - op.create_table('denorm_plants_hydro_ferc1', - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), - sa.Column('avg_num_employees', sa.Float(), nullable=True), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), - sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), - sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), - sa.Column('net_capacity_adverse_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the least favorable operating conditions, in megawatts.'), - sa.Column('net_capacity_favorable_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the most favorable operating conditions, in megawatts.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), - sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), - sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), - sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), - sa.Column('opex_hydraulic', sa.Float(), nullable=True, comment='Production expenses: hydraulic expenses (USD).'), - sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), - sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), - sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_type', sa.Text(), nullable=True), - sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_hydro_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_plants_hydro_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_hydro_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('record_id', name=op.f('pk_denorm_plants_hydro_ferc1')) - ) - op.create_table('denorm_plants_pumped_storage_ferc1', - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), - sa.Column('avg_num_employees', sa.Float(), nullable=True), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('capex_equipment_electric', sa.Float(), nullable=True, comment='Cost of plant: accessory electric equipment (USD).'), - sa.Column('capex_equipment_misc', sa.Float(), nullable=True, comment='Cost of plant: miscellaneous power plant equipment (USD).'), - sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), - sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('capex_wheels_turbines_generators', sa.Float(), nullable=True, comment='Cost of plant: water wheels, turbines, and generators (USD).'), - sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('energy_used_for_pumping_mwh', sa.Float(), nullable=True, comment='Energy used for pumping, in megawatt-hours.'), - sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('net_load_mwh', sa.Float(), nullable=True, comment='Net output for load (net generation - energy used for pumping) in megawatt-hours.'), - sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), - sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), - sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), - sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), - sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), - sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), - sa.Column('opex_production_before_pumping', sa.Float(), nullable=True, comment='Total production expenses before pumping (USD).'), - sa.Column('opex_pumped_storage', sa.Float(), nullable=True, comment='Production expenses: pumped storage (USD).'), - sa.Column('opex_pumping', sa.Float(), nullable=True, comment='Production expenses: We are here to PUMP YOU UP! (USD).'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), - sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), - sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_pumped_storage_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_plants_pumped_storage_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_pumped_storage_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('record_id', name=op.f('pk_denorm_plants_pumped_storage_ferc1')) - ) - op.create_table('denorm_plants_small_ferc1', - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('water_source', sa.Text(), nullable=True, comment='Name of water source associated with the plant.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('fuel_type', sa.Text(), nullable=True), - sa.Column('license_id_ferc1', sa.Integer(), nullable=True, comment='FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), - sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('plant_type', sa.Text(), nullable=True), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_small_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_plants_small_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_small_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('record_id', name=op.f('pk_denorm_plants_small_ferc1')) - ) - op.create_table('denorm_plants_steam_ferc1', + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('balancing_authority_code_eia_consistent_rate', sa.Float(), nullable=True, comment='Percentage consistency of balancing authority code across entity records.'), + sa.ForeignKeyConstraint(['balancing_authority_code_eia'], ['core_eia__codes_balancing_authorities.code'], name=op.f('fk_out_eia__yearly_plants_balancing_authority_code_eia_core_eia__codes_balancing_authorities')), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia__yearly_plants_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['core_eia860__scd_plants.plant_id_eia', 'core_eia860__scd_plants.report_date'], name=op.f('fk_out_eia__yearly_plants_plant_id_eia_core_eia860__scd_plants')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia__yearly_plants_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['reporting_frequency_code'], ['core_eia__codes_reporting_frequencies.code'], name=op.f('fk_out_eia__yearly_plants_reporting_frequency_code_core_eia__codes_reporting_frequencies')), + sa.ForeignKeyConstraint(['sector_id_eia'], ['core_eia__codes_sector_consolidated.code'], name=op.f('fk_out_eia__yearly_plants_sector_id_eia_core_eia__codes_sector_consolidated')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia__yearly_plants_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia__yearly_plants_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', name=op.f('pk_out_eia__yearly_plants')) + ) + op.create_table('out_ferc1__yearly_all_plants', sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), @@ -2636,13 +2969,13 @@ def upgrade() -> None: sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), - sa.Column('opex_fuel_per_mwh', sa.Float(), nullable=True, comment='Production expenses: fuel (USD) per megawatt-hour (Mwh).'), + sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), sa.Column('opex_misc_power', sa.Float(), nullable=True, comment='Miscellaneous steam (or nuclear) expenses.'), sa.Column('opex_misc_steam', sa.Float(), nullable=True, comment='Maintenance of miscellaneous steam (or nuclear) plant.'), sa.Column('opex_nonfuel_per_mwh', sa.Float(), nullable=True, comment='Investments in non-fuel production expenses per Mwh.'), sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), - sa.Column('opex_plants', sa.Float(), nullable=True, comment='Maintenance of electrical plant.'), + sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), sa.Column('opex_production_total', sa.Float(), nullable=True, comment='Total operating expenses.'), sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), sa.Column('opex_steam', sa.Float(), nullable=True, comment='Steam expenses.'), @@ -2656,325 +2989,284 @@ def upgrade() -> None: sa.Column('plant_type', sa.Text(), nullable=True), sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), sa.Column('water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when limited by condenser water.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_steam_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_plants_steam_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_steam_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('record_id', name=op.f('pk_denorm_plants_steam_ferc1')) - ) - op.create_table('denorm_plants_utilities_eia', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['plants_eia860.plant_id_eia', 'plants_eia860.report_date'], name=op.f('fk_denorm_plants_utilities_eia_plant_id_eia_plants_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_utilities_eia_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_utilities_eia_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'utility_id_eia', name=op.f('pk_denorm_plants_utilities_eia')) - ) - op.create_table('denorm_plants_utilities_ferc1', - sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=False, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_plants_utilities_ferc1_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_denorm_plants_utilities_ferc1_utility_id_ferc1_plants_ferc1')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_plants_utilities_ferc1_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('utility_id_ferc1', 'plant_name_ferc1', name=op.f('pk_denorm_plants_utilities_ferc1')) - ) - op.create_table('fuel_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_units', sa.Enum('mmbtu', 'gramsU', 'kg', 'mwhth', 'kgal', 'bbl', 'klbs', 'mcf', 'gal', 'mwdth', 'btu', 'ton'), nullable=True, comment='Reported unit of measure for fuel.'), - sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), - sa.Column('fuel_cost_per_unit_burned', sa.Float(), nullable=True, comment='Average cost of fuel consumed in the report year per reported fuel unit (USD).'), - sa.Column('fuel_cost_per_unit_delivered', sa.Float(), nullable=True, comment='Average cost of fuel delivered in the report year per reported fuel unit (USD).'), sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_fuel_ferc1_utility_id_ferc1_plants_ferc1')) - ) - op.create_table('generators_eia860', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), - sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), - sa.Column('ownership_code', sa.Text(), nullable=True, comment='Identifies the ownership for each generator.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('summer_capacity_mw', sa.Float(), nullable=True, comment='The net summer capacity.'), - sa.Column('summer_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the summer capacity value was an estimate'), - sa.Column('winter_capacity_mw', sa.Float(), nullable=True, comment='The net winter capacity.'), - sa.Column('winter_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the winter capacity value was an estimate'), - sa.Column('net_capacity_mwdc', sa.Float(), nullable=True, comment='Generation capacity in megawatts of direct current that is subject to a net metering agreement. Typically used for behind-the-meter solar PV.'), - sa.Column('energy_storage_capacity_mwh', sa.Float(), nullable=True, comment='Energy storage capacity in MWh (e.g. for batteries).'), - sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), - sa.Column('energy_source_code_2', sa.Text(), nullable=True, comment='The code representing the second most predominant type of energy that fuels the generator'), - sa.Column('energy_source_code_3', sa.Text(), nullable=True, comment='The code representing the third most predominant type of energy that fuels the generator'), - sa.Column('energy_source_code_4', sa.Text(), nullable=True, comment='The code representing the fourth most predominant type of energy that fuels the generator'), - sa.Column('energy_source_code_5', sa.Text(), nullable=True, comment='The code representing the fifth most predominant type of energy that fuels the generator'), - sa.Column('energy_source_code_6', sa.Text(), nullable=True, comment='The code representing the sixth most predominant type of energy that fuels the generator'), - sa.Column('energy_source_1_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 1.'), - sa.Column('energy_source_1_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 1.'), - sa.Column('energy_source_1_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 1.'), - sa.Column('energy_source_2_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 2.'), - sa.Column('energy_source_2_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 2.'), - sa.Column('energy_source_2_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 2.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('multiple_fuels', sa.Boolean(), nullable=True, comment='Can the generator burn multiple fuels?'), - sa.Column('deliver_power_transgrid', sa.Boolean(), nullable=True, comment='Indicate whether the generator can deliver power to the transmission grid.'), - sa.Column('distributed_generation', sa.Boolean(), nullable=True, comment='Whether the generator is considered distributed generation'), - sa.Column('syncronized_transmission_grid', sa.Boolean(), nullable=True, comment='Indicates whether standby generators (SB status) can be synchronized to the grid.'), - sa.Column('turbines_num', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), - sa.Column('planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are any planned capacity uprates/derates, repowering, other modifications, or generator retirements scheduled for the next 5 years.'), - sa.Column('planned_net_summer_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in summer capacity expected to be realized from the modification to the equipment.'), - sa.Column('planned_net_winter_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in winter capacity expected to be realized from the uprate modification to the equipment.'), - sa.Column('planned_uprate_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the uprate modification.'), - sa.Column('planned_net_summer_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in summer capacity expected to be realized from the derate modification to the equipment.'), - sa.Column('planned_net_winter_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in winter capacity expected to be realized from the derate modification to the equipment.'), - sa.Column('planned_derate_date', sa.Date(), nullable=True, comment='Planned effective month that the generator is scheduled to enter operation after the derate modification.'), - sa.Column('planned_new_prime_mover_code', sa.Text(), nullable=True, comment='New prime mover for the planned repowered generator.'), - sa.Column('planned_energy_source_code_1', sa.Text(), nullable=True, comment='New energy source code for the planned repowered generator.'), - sa.Column('planned_repower_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the repowering is complete.'), - sa.Column('other_planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are there other modifications planned for the generator.'), - sa.Column('other_modifications_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter commercial operation after any other planned modification is complete.'), - sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), - sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), - sa.Column('startup_source_code_1', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), - sa.Column('startup_source_code_2', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), - sa.Column('startup_source_code_3', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), - sa.Column('startup_source_code_4', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), - sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), - sa.Column('turbines_inverters_hydrokinetics', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), - sa.Column('time_cold_shutdown_full_load_code', sa.Text(), nullable=True, comment='The minimum amount of time required to bring the unit to full load from shutdown.'), - sa.Column('planned_new_capacity_mw', sa.Float(), nullable=True, comment='The expected new namplate capacity for the generator.'), - sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), - sa.Column('switch_oil_gas', sa.Boolean(), nullable=True, comment='Indicates whether the generator switch between oil and natural gas.'), - sa.Column('nameplate_power_factor', sa.Float(), nullable=True, comment='The nameplate power factor of the generator.'), - sa.Column('minimum_load_mw', sa.Float(), nullable=True, comment='The minimum load at which the generator can operate at continuosuly.'), - sa.Column('uprate_derate_during_year', sa.Boolean(), nullable=True, comment='Was an uprate or derate completed on this generator during the reporting year?'), - sa.Column('uprate_derate_completed_date', sa.Date(), nullable=True, comment='The date when the uprate or derate was completed.'), - sa.Column('current_planned_generator_operating_date', sa.Date(), nullable=True, comment='The most recently updated effective date on which the generator is scheduled to start operation'), - sa.Column('summer_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated summer capacity (in MWh).'), - sa.Column('winter_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated winter capacity (in MWh).'), - sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), - sa.Column('owned_by_non_utility', sa.Boolean(), nullable=True, comment='Whether any part of generator is owned by a nonutilty'), - sa.Column('reactive_power_output_mvar', sa.Float(), nullable=True, comment='Reactive Power Output (MVAr)'), - sa.Column('ferc_qualifying_facility', sa.Boolean(), nullable=True, comment='Indicatates whether or not a generator is a qualifying FERC cogeneation facility.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_generators_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['energy_source_1_transport_1'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_generators_eia860_energy_source_1_transport_1_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_1_transport_2'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_generators_eia860_energy_source_1_transport_2_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_1_transport_3'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_generators_eia860_energy_source_1_transport_3_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_2_transport_1'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_generators_eia860_energy_source_2_transport_1_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_2_transport_2'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_generators_eia860_energy_source_2_transport_2_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_2_transport_3'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_generators_eia860_energy_source_2_transport_3_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_2'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_energy_source_code_2_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_3'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_energy_source_code_3_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_4'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_energy_source_code_4_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_5'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_energy_source_code_5_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_6'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_energy_source_code_6_energy_sources_eia')), - sa.ForeignKeyConstraint(['operational_status_code'], ['operational_status_eia.code'], name=op.f('fk_generators_eia860_operational_status_code_operational_status_eia')), - sa.ForeignKeyConstraint(['planned_energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_planned_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['planned_new_prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_generators_eia860_planned_new_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id'], ['generators_entity_eia.plant_id_eia', 'generators_entity_eia.generator_id'], name=op.f('fk_generators_eia860_plant_id_eia_generators_entity_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['plants_eia860.plant_id_eia', 'plants_eia860.report_date'], name=op.f('fk_generators_eia860_plant_id_eia_plants_eia860')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_generators_eia860_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['startup_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_startup_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['startup_source_code_2'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_startup_source_code_2_energy_sources_eia')), - sa.ForeignKeyConstraint(['startup_source_code_3'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_startup_source_code_3_energy_sources_eia')), - sa.ForeignKeyConstraint(['startup_source_code_4'], ['energy_sources_eia.code'], name=op.f('fk_generators_eia860_startup_source_code_4_energy_sources_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_generators_eia860_utility_id_eia_utilities_eia860')), - sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_generators_eia860')) - ) - op.create_table('heat_rate_by_unit_yearly', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=False, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'report_date'], ['plants_eia860.plant_id_eia', 'plants_eia860.report_date'], name=op.f('fk_heat_rate_by_unit_yearly_plant_id_eia_plants_eia860')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'unit_id_pudl', name=op.f('pk_heat_rate_by_unit_yearly')) - ) - op.create_table('plants_hydro_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), - sa.Column('plant_type', sa.Enum('run_of_river_with_storage', 'storage', 'hydro', 'run_of_river', 'na_category'), nullable=True), - sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), - sa.Column('net_capacity_favorable_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the most favorable operating conditions, in megawatts.'), - sa.Column('net_capacity_adverse_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the least favorable operating conditions, in megawatts.'), - sa.Column('avg_num_employees', sa.Float(), nullable=True), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), - sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('fuel_type', sa.Text(), nullable=True), + sa.Column('license_id_ferc1', sa.Integer(), nullable=True, comment='FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.'), + sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), - sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), - sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), - sa.Column('opex_hydraulic', sa.Float(), nullable=True, comment='Production expenses: hydraulic expenses (USD).'), - sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), - sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), - sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), - sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('net_capacity_adverse_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the least favorable operating conditions, in megawatts.'), + sa.Column('net_capacity_favorable_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the most favorable operating conditions, in megawatts.'), sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), - sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), + sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), + sa.Column('opex_hydraulic', sa.Float(), nullable=True, comment='Production expenses: hydraulic expenses (USD).'), sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_plants_hydro_ferc1_utility_id_ferc1_plants_ferc1')) - ) - op.create_table('plants_pumped_storage_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('project_num', sa.Integer(), nullable=True, comment='FERC Licensed Project Number.'), - sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), - sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), - sa.Column('avg_num_employees', sa.Float(), nullable=True), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('energy_used_for_pumping_mwh', sa.Float(), nullable=True, comment='Energy used for pumping, in megawatt-hours.'), - sa.Column('net_load_mwh', sa.Float(), nullable=True, comment='Net output for load (net generation - energy used for pumping) in megawatt-hours.'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), - sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), - sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), - sa.Column('capex_wheels_turbines_generators', sa.Float(), nullable=True, comment='Cost of plant: water wheels, turbines, and generators (USD).'), + sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), + sa.Column('ferc_license_id', sa.Text(), nullable=True, comment='The FERC license ID of a project.'), sa.Column('capex_equipment_electric', sa.Float(), nullable=True, comment='Cost of plant: accessory electric equipment (USD).'), sa.Column('capex_equipment_misc', sa.Float(), nullable=True, comment='Cost of plant: miscellaneous power plant equipment (USD).'), - sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), - sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), - sa.Column('opex_pumped_storage', sa.Float(), nullable=True, comment='Production expenses: pumped storage (USD).'), - sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), - sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), - sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), - sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), - sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), - sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), - sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), + sa.Column('capex_wheels_turbines_generators', sa.Float(), nullable=True, comment='Cost of plant: water wheels, turbines, and generators (USD).'), + sa.Column('energy_used_for_pumping_mwh', sa.Float(), nullable=True, comment='Energy used for pumping, in megawatt-hours.'), + sa.Column('net_load_mwh', sa.Float(), nullable=True, comment='Net output for load (net generation - energy used for pumping) in megawatt-hours.'), sa.Column('opex_production_before_pumping', sa.Float(), nullable=True, comment='Total production expenses before pumping (USD).'), + sa.Column('opex_pumped_storage', sa.Float(), nullable=True, comment='Production expenses: pumped storage (USD).'), sa.Column('opex_pumping', sa.Float(), nullable=True, comment='Production expenses: We are here to PUMP YOU UP! (USD).'), - sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), - sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_plants_pumped_storage_ferc1_utility_id_ferc1_plants_ferc1')) + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_ferc1__yearly_all_plants_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_out_ferc1__yearly_all_plants_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_all_plants_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id', name=op.f('pk_out_ferc1__yearly_all_plants')) ) - op.create_table('plants_small_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + op.create_table('out_ferc1__yearly_steam_plants_fuel_by_plant_sched402', + sa.Column('report_year', sa.Integer(), nullable=False, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=False, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=False, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('coal_fraction_cost', sa.Float(), nullable=True, comment='Coal cost as a percentage of overall fuel cost.'), + sa.Column('coal_fraction_mmbtu', sa.Float(), nullable=True, comment='Coal heat content as a percentage of overall fuel heat content (mmBTU).'), + sa.Column('fuel_cost', sa.Float(), nullable=True, comment='Total fuel cost for plant (in $USD).'), + sa.Column('fuel_mmbtu', sa.Float(), nullable=True, comment='Total heat content for plant (in MMBtu).'), + sa.Column('gas_fraction_cost', sa.Float(), nullable=True, comment='Natural gas cost as a percentage of overall fuel cost.'), + sa.Column('gas_fraction_mmbtu', sa.Float(), nullable=True, comment='Natural gas heat content as a percentage of overall fuel heat content (MMBtu).'), + sa.Column('nuclear_fraction_cost', sa.Float(), nullable=True, comment='Nuclear cost as a percentage of overall fuel cost.'), + sa.Column('nuclear_fraction_mmbtu', sa.Float(), nullable=True, comment='Nuclear heat content as a percentage of overall fuel heat content (MMBtu).'), + sa.Column('oil_fraction_cost', sa.Float(), nullable=True, comment='Oil cost as a percentage of overall fuel cost.'), + sa.Column('oil_fraction_mmbtu', sa.Float(), nullable=True, comment='Oil heat content as a percentage of overall fuel heat content (MMBtu).'), + sa.Column('primary_fuel_by_cost', sa.Text(), nullable=True, comment='Primary fuel for plant as a percentage of cost.'), + sa.Column('primary_fuel_by_mmbtu', sa.Text(), nullable=True, comment='Primary fuel for plant as a percentage of heat content.'), + sa.Column('waste_fraction_cost', sa.Float(), nullable=True, comment='Waste-heat cost as a percentage of overall fuel cost.'), + sa.Column('waste_fraction_mmbtu', sa.Float(), nullable=True, comment='Waste-heat heat content as a percentage of overall fuel heat content (MMBtu).'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_ferc1__yearly_steam_plants_fuel_by_plant_sched402_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_out_ferc1__yearly_steam_plants_fuel_by_plant_sched402_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_steam_plants_fuel_by_plant_sched402_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_year', 'utility_id_ferc1', 'plant_name_ferc1', name=op.f('pk_out_ferc1__yearly_steam_plants_fuel_by_plant_sched402')) + ) + op.create_table('out_ferc1__yearly_steam_plants_fuel_sched402', sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('plant_type', sa.Text(), nullable=True), - sa.Column('license_id_ferc1', sa.Integer(), nullable=True, comment='FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.'), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), - sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), - sa.Column('fuel_type', sa.Text(), nullable=True), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_total_cost', sa.Float(), nullable=True, comment='Total cost of consumed fuel.'), + sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_plants_small_ferc1_utility_id_ferc1_plants_ferc1')) + sa.Column('fuel_cost_per_unit_burned', sa.Float(), nullable=True, comment='Average cost of fuel consumed in the report year per reported fuel unit (USD).'), + sa.Column('fuel_cost_per_unit_delivered', sa.Float(), nullable=True, comment='Average cost of fuel delivered in the report year per reported fuel unit (USD).'), + sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_units', sa.Enum('mmbtu', 'gramsU', 'kg', 'mwhth', 'kgal', 'bbl', 'klbs', 'mcf', 'gal', 'mwdth', 'btu', 'ton'), nullable=True, comment='Reported unit of measure for fuel.'), + sa.Column('record_id', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_ferc1__yearly_steam_plants_fuel_sched402_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_out_ferc1__yearly_steam_plants_fuel_sched402_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_ferc1__yearly_steam_plants_fuel_sched402_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id', name=op.f('pk_out_ferc1__yearly_steam_plants_fuel_sched402')) ) - op.create_table('plants_steam_ferc1', - sa.Column('record_id', sa.Text(), nullable=True, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), - sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), - sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), - sa.Column('plant_id_ferc1', sa.Integer(), nullable=True, comment='Algorithmically assigned PUDL FERC Plant ID. WARNING: NOT STABLE BETWEEN PUDL DB INITIALIZATIONS.'), - sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), - sa.Column('plant_type', sa.Enum('wind', 'photovoltaic', 'na_category', 'solar_thermal', 'combined_cycle', 'internal_combustion', 'combustion_turbine', 'geothermal', 'steam', 'nuclear'), nullable=True), - sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), - sa.Column('construction_year', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), - sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + op.create_table('_out_eia__yearly_capacity_factor_by_generator', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), - sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), - sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), - sa.Column('water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when limited by condenser water.'), - sa.Column('not_water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when not limited by condenser water.'), - sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk__out_eia__yearly_capacity_factor_by_generator_plant_id_eia_core_eia860__scd_generators')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__yearly_capacity_factor_by_generator')) + ) + op.create_table('_out_eia__yearly_derived_generator_attributes', + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), - sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), - sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), - sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), - sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), - sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), - sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), - sa.Column('opex_coolants', sa.Float(), nullable=True, comment='Cost of coolants and water (nuclear plants only)'), - sa.Column('opex_steam', sa.Float(), nullable=True, comment='Steam expenses.'), - sa.Column('opex_steam_other', sa.Float(), nullable=True, comment='Steam from other sources.'), - sa.Column('opex_transfer', sa.Float(), nullable=True, comment='Steam transferred (Credit).'), - sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), - sa.Column('opex_misc_power', sa.Float(), nullable=True, comment='Miscellaneous steam (or nuclear) expenses.'), - sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), - sa.Column('opex_allowances', sa.Float(), nullable=True, comment='Allowances.'), - sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), - sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), - sa.Column('opex_boiler', sa.Float(), nullable=True, comment='Maintenance of boiler (or reactor) plant.'), - sa.Column('opex_plants', sa.Float(), nullable=True, comment='Maintenance of electrical plant.'), - sa.Column('opex_misc_steam', sa.Float(), nullable=True, comment='Maintenance of miscellaneous steam (or nuclear) plant.'), - sa.Column('opex_production_total', sa.Float(), nullable=True, comment='Total operating expenses.'), - sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), - sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), - sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['plants_ferc1.utility_id_ferc1', 'plants_ferc1.plant_name_ferc1'], name=op.f('fk_plants_steam_ferc1_utility_id_ferc1_plants_ferc1')) + sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk__out_eia__yearly_derived_generator_attributes_plant_id_eia_core_eia860__scd_generators')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__yearly_derived_generator_attributes')) ) - op.create_table('boiler_cooling_assn_eia860', + op.create_table('_out_eia__yearly_fuel_cost_by_generator', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), - sa.Column('cooling_id_eia', sa.Text(), nullable=False, comment='The cooling system identification number reported to EIA.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk__out_eia__yearly_fuel_cost_by_generator_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_eia__yearly_fuel_cost_by_generator_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk__out_eia__yearly_fuel_cost_by_generator_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_eia__yearly_fuel_cost_by_generator_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__yearly_fuel_cost_by_generator')) + ) + op.create_table('_out_eia__yearly_generators', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('associated_combined_heat_power', sa.Boolean(), nullable=True, comment='Indicates whether the generator is associated with a combined heat and power system'), + sa.Column('bga_source', sa.Text(), nullable=True, comment='The source from where the unit_id_pudl is compiled. The unit_id_pudl comes directly from EIA 860, or string association (which looks at all the boilers and generators that are not associated with a unit and tries to find a matching string in the respective collection of boilers or generator), or from a unit connection (where the unit_id_eia is employed to find additional boiler generator connections).'), + sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), + sa.Column('city', sa.Text(), nullable=True), + sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), + sa.Column('county', sa.Text(), nullable=True, comment='County name.'), + sa.Column('current_planned_generator_operating_date', sa.Date(), nullable=True, comment='The most recently updated effective date on which the generator is scheduled to start operation'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_boiler_cooling_assn_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['boilers_eia860.plant_id_eia', 'boilers_eia860.boiler_id', 'boilers_eia860.report_date'], name=op.f('fk_boiler_cooling_assn_eia860_plant_id_eia_boilers_eia860')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'boiler_id', 'cooling_id_eia', name=op.f('pk_boiler_cooling_assn_eia860')) + sa.Column('deliver_power_transgrid', sa.Boolean(), nullable=True, comment='Indicate whether the generator can deliver power to the transmission grid.'), + sa.Column('distributed_generation', sa.Boolean(), nullable=True, comment='Whether the generator is considered distributed generation'), + sa.Column('duct_burners', sa.Boolean(), nullable=True, comment='Indicates whether the unit has duct-burners for supplementary firing of the turbine exhaust gas'), + sa.Column('energy_source_1_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 1.'), + sa.Column('energy_source_1_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 1.'), + sa.Column('energy_source_1_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 1.'), + sa.Column('energy_source_2_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 2.'), + sa.Column('energy_source_2_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 2.'), + sa.Column('energy_source_2_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 2.'), + sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), + sa.Column('energy_source_code_2', sa.Text(), nullable=True, comment='The code representing the second most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_3', sa.Text(), nullable=True, comment='The code representing the third most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_4', sa.Text(), nullable=True, comment='The code representing the fourth most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_5', sa.Text(), nullable=True, comment='The code representing the fifth most predominant type of energy that fuels the generator'), + sa.Column('energy_source_code_6', sa.Text(), nullable=True, comment='The code representing the sixth most predominant type of energy that fuels the generator'), + sa.Column('energy_storage_capacity_mwh', sa.Float(), nullable=True, comment='Energy storage capacity in MWh (e.g. for batteries).'), + sa.Column('ferc_qualifying_facility', sa.Boolean(), nullable=True, comment='Indicatates whether or not a generator is a qualifying FERC cogeneation facility.'), + sa.Column('fluidized_bed_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses fluidized bed technology'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), + sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), + sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), + sa.Column('latitude', sa.Float(), nullable=True, comment="Latitude of the plant's location, in degrees."), + sa.Column('longitude', sa.Float(), nullable=True, comment="Longitude of the plant's location, in degrees."), + sa.Column('minimum_load_mw', sa.Float(), nullable=True, comment='The minimum load at which the generator can operate at continuosuly.'), + sa.Column('multiple_fuels', sa.Boolean(), nullable=True, comment='Can the generator burn multiple fuels?'), + sa.Column('nameplate_power_factor', sa.Float(), nullable=True, comment='The nameplate power factor of the generator.'), + sa.Column('net_capacity_mwdc', sa.Float(), nullable=True, comment='Generation capacity in megawatts of direct current that is subject to a net metering agreement. Typically used for behind-the-meter solar PV.'), + sa.Column('operating_switch', sa.Text(), nullable=True, comment='Indicates whether the fuel switching generator can switch when operating'), + sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), + sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), + sa.Column('original_planned_generator_operating_date', sa.Date(), nullable=True, comment='The date the generator was originally scheduled to be operational'), + sa.Column('other_combustion_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses other combustion technologies'), + sa.Column('other_modifications_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter commercial operation after any other planned modification is complete.'), + sa.Column('other_planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are there other modifications planned for the generator.'), + sa.Column('owned_by_non_utility', sa.Boolean(), nullable=True, comment='Whether any part of generator is owned by a nonutilty'), + sa.Column('ownership_code', sa.Text(), nullable=True, comment='Identifies the ownership for each generator.'), + sa.Column('planned_derate_date', sa.Date(), nullable=True, comment='Planned effective month that the generator is scheduled to enter operation after the derate modification.'), + sa.Column('planned_energy_source_code_1', sa.Text(), nullable=True, comment='New energy source code for the planned repowered generator.'), + sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), + sa.Column('planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are any planned capacity uprates/derates, repowering, other modifications, or generator retirements scheduled for the next 5 years.'), + sa.Column('planned_net_summer_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in summer capacity expected to be realized from the derate modification to the equipment.'), + sa.Column('planned_net_summer_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in summer capacity expected to be realized from the modification to the equipment.'), + sa.Column('planned_net_winter_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in winter capacity expected to be realized from the derate modification to the equipment.'), + sa.Column('planned_net_winter_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in winter capacity expected to be realized from the uprate modification to the equipment.'), + sa.Column('planned_new_capacity_mw', sa.Float(), nullable=True, comment='The expected new namplate capacity for the generator.'), + sa.Column('planned_new_prime_mover_code', sa.Text(), nullable=True, comment='New prime mover for the planned repowered generator.'), + sa.Column('planned_repower_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the repowering is complete.'), + sa.Column('planned_uprate_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the uprate modification.'), + sa.Column('previously_canceled', sa.Boolean(), nullable=True, comment='Indicates whether the generator was previously reported as indefinitely postponed or canceled'), + sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('pulverized_coal_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses pulverized coal technology'), + sa.Column('reactive_power_output_mvar', sa.Float(), nullable=True, comment='Reactive Power Output (MVAr)'), + sa.Column('rto_iso_lmp_node_id', sa.Text(), nullable=True, comment='The designation used to identify the price node in RTO/ISO Locational Marginal Price reports'), + sa.Column('rto_iso_location_wholesale_reporting_id', sa.Text(), nullable=True, comment='The designation used to report ths specific location of the wholesale sales transactions to FERC for the Electric Quarterly Report'), + sa.Column('solid_fuel_gasification', sa.Boolean(), nullable=True, comment='Indicates whether the generator is part of a solid fuel gasification system'), + sa.Column('startup_source_code_1', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_2', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_3', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('startup_source_code_4', sa.Text(), nullable=True, comment='The code representing the first, second, third or fourth start-up and flame stabilization energy source used by the combustion unit(s) associated with this generator.'), + sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), + sa.Column('stoker_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses stoker technology'), + sa.Column('street_address', sa.Text(), nullable=True), + sa.Column('subcritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses subcritical technology'), + sa.Column('summer_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the summer capacity value was an estimate'), + sa.Column('summer_capacity_mw', sa.Float(), nullable=True, comment='The net summer capacity.'), + sa.Column('summer_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated summer capacity (in MWh).'), + sa.Column('supercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses supercritical technology'), + sa.Column('switch_oil_gas', sa.Boolean(), nullable=True, comment='Indicates whether the generator switch between oil and natural gas.'), + sa.Column('syncronized_transmission_grid', sa.Boolean(), nullable=True, comment='Indicates whether standby generators (SB status) can be synchronized to the grid.'), + sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), + sa.Column('time_cold_shutdown_full_load_code', sa.Text(), nullable=True, comment='The minimum amount of time required to bring the unit to full load from shutdown.'), + sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), + sa.Column('topping_bottoming_code', sa.Text(), nullable=True, comment='If the generator is associated with a combined heat and power system, indicates whether the generator is part of a topping cycle or a bottoming cycle'), + sa.Column('turbines_inverters_hydrokinetics', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), + sa.Column('turbines_num', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), + sa.Column('ultrasupercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses ultra-supercritical technology'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('uprate_derate_completed_date', sa.Date(), nullable=True, comment='The date when the uprate or derate was completed.'), + sa.Column('uprate_derate_during_year', sa.Boolean(), nullable=True, comment='Was an uprate or derate completed on this generator during the reporting year?'), + sa.Column('winter_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the winter capacity value was an estimate'), + sa.Column('winter_capacity_mw', sa.Float(), nullable=True, comment='The net winter capacity.'), + sa.Column('winter_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated winter capacity (in MWh).'), + sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk__out_eia__yearly_generators_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_1_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_1_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_1_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_1_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_2_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_2_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_2_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_5'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_code_5_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_6'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_energy_source_code_6_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['operational_status_code'], ['core_eia__codes_operational_status.code'], name=op.f('fk__out_eia__yearly_generators_operational_status_code_core_eia__codes_operational_status')), + sa.ForeignKeyConstraint(['planned_energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_planned_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['planned_new_prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk__out_eia__yearly_generators_planned_new_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk__out_eia__yearly_generators_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk__out_eia__yearly_generators_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk__out_eia__yearly_generators_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['startup_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_startup_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_startup_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_startup_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk__out_eia__yearly_generators_startup_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk__out_eia__yearly_generators_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk__out_eia__yearly_generators_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk__out_eia__yearly_generators')) + ) + op.create_table('_out_eia__yearly_heat_rate_by_generator', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), + sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk__out_eia__yearly_heat_rate_by_generator_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk__out_eia__yearly_heat_rate_by_generator_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk__out_eia__yearly_heat_rate_by_generator')) ) - op.create_table('boiler_emissions_control_equipment_assn_eia860', + op.create_table('core_eia860__assn_boiler_cooling', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), - sa.Column('emission_control_id_type', sa.Text(), nullable=False, comment='The type of emissions control id: so2, nox, particulate, or mercury.'), - sa.Column('emission_control_id_eia', sa.Text(), nullable=False, comment="The emission control ID used to collect so2, nox, particulate, and mercury emissions data. This column should be used in conjunction with emissions_control_type as it's not guaranteed to be unique."), + sa.Column('cooling_id_eia', sa.Text(), nullable=False, comment='The cooling system identification number reported to EIA.'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_boiler_emissions_control_equipment_assn_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['boilers_eia860.plant_id_eia', 'boilers_eia860.boiler_id', 'boilers_eia860.report_date'], name=op.f('fk_boiler_emissions_control_equipment_assn_eia860_plant_id_eia_boilers_eia860')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'boiler_id', 'emission_control_id_type', 'emission_control_id_eia', name=op.f('pk_boiler_emissions_control_equipment_assn_eia860')) + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__assn_boiler_cooling_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['core_eia860__scd_boilers.plant_id_eia', 'core_eia860__scd_boilers.boiler_id', 'core_eia860__scd_boilers.report_date'], name=op.f('fk_core_eia860__assn_boiler_cooling_plant_id_eia_core_eia860__scd_boilers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'boiler_id', 'cooling_id_eia', name=op.f('pk_core_eia860__assn_boiler_cooling')) ) - op.create_table('boiler_generator_assn_eia860', + op.create_table('core_eia860__assn_boiler_generator', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), @@ -2985,14 +3277,14 @@ def upgrade() -> None: sa.Column('steam_plant_type_code', sa.Integer(), nullable=True, comment='Code that describes types of steam plants from EIA 860. See steam_plant_types_eia table for more details.'), sa.Column('bga_source', sa.Text(), nullable=True, comment='The source from where the unit_id_pudl is compiled. The unit_id_pudl comes directly from EIA 860, or string association (which looks at all the boilers and generators that are not associated with a unit and tries to find a matching string in the respective collection of boilers or generator), or from a unit connection (where the unit_id_eia is employed to find additional boiler generator connections).'), sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['boiler_generator_assn_type_code'], ['boiler_generator_assn_types_eia.code'], name=op.f('fk_boiler_generator_assn_eia860_boiler_generator_assn_type_code_boiler_generator_assn_types_eia')), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_boiler_generator_assn_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['boilers_eia860.plant_id_eia', 'boilers_eia860.boiler_id', 'boilers_eia860.report_date'], name=op.f('fk_boiler_generator_assn_eia860_plant_id_eia_boilers_eia860')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_boiler_generator_assn_eia860_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['steam_plant_type_code'], ['steam_plant_types_eia.code'], name=op.f('fk_boiler_generator_assn_eia860_steam_plant_type_code_steam_plant_types_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'generator_id', 'boiler_id', name=op.f('pk_boiler_generator_assn_eia860')) - ) - op.create_table('boiler_stack_flue_assn_eia860', + sa.ForeignKeyConstraint(['boiler_generator_assn_type_code'], ['core_eia__codes_boiler_generator_assn_types.code'], name=op.f('fk_core_eia860__assn_boiler_generator_boiler_generator_assn_type_code_core_eia__codes_boiler_generator_assn_types')), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__assn_boiler_generator_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['core_eia860__scd_boilers.plant_id_eia', 'core_eia860__scd_boilers.boiler_id', 'core_eia860__scd_boilers.report_date'], name=op.f('fk_core_eia860__assn_boiler_generator_plant_id_eia_core_eia860__scd_boilers')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_core_eia860__assn_boiler_generator_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['steam_plant_type_code'], ['core_eia__codes_steam_plant_types.code'], name=op.f('fk_core_eia860__assn_boiler_generator_steam_plant_type_code_core_eia__codes_steam_plant_types')), + sa.PrimaryKeyConstraint('plant_id_eia', 'report_date', 'generator_id', 'boiler_id', name=op.f('pk_core_eia860__assn_boiler_generator')) + ) + op.create_table('core_eia860__assn_boiler_stack_flue', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), @@ -3000,20 +3292,62 @@ def upgrade() -> None: sa.Column('flue_id_eia', sa.Text(), nullable=True, comment='The flue identification value reported to EIA. The flue is a duct, pipe, or opening that transports exhast gases through the stack. This field was reported in conjunction with stack_id_eia until 2013 when stack_flue_id_eia took their place.'), sa.Column('stack_flue_id_eia', sa.Text(), nullable=True, comment='The stack or flue identification value reported to EIA. This denotes the place where emissions from the combusion process are released into the atmosphere. Prior to 2013, this was reported as `stack_id_eia` and `flue_id_eia`.'), sa.Column('stack_flue_id_pudl', sa.Text(), nullable=False, comment='A stack and/or flue identification value created by PUDL for use as part of the primary key for the stack flue equipment and boiler association tables. For 2013 and onward, this value is equal to the value for stack_flue_id_eia. Prior to 2013, this value is equal to the value for stack_id_eia and the value for flue_id_eia seperated by an underscore or just the stack_flue_eia in cases where flue_id_eia is NA.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['boilers_eia860.plant_id_eia', 'boilers_eia860.boiler_id', 'boilers_eia860.report_date'], name=op.f('fk_boiler_stack_flue_assn_eia860_plant_id_eia_boilers_eia860')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'boiler_id', 'stack_flue_id_pudl', name=op.f('pk_boiler_stack_flue_assn_eia860')) + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['core_eia860__scd_boilers.plant_id_eia', 'core_eia860__scd_boilers.boiler_id', 'core_eia860__scd_boilers.report_date'], name=op.f('fk_core_eia860__assn_boiler_stack_flue_plant_id_eia_core_eia860__scd_boilers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'boiler_id', 'stack_flue_id_pudl', name=op.f('pk_core_eia860__assn_boiler_stack_flue')) + ) + op.create_table('core_eia860__assn_yearly_boiler_emissions_control_equipment', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('boiler_id', sa.Text(), nullable=False, comment='Alphanumeric boiler ID.'), + sa.Column('emission_control_id_type', sa.Text(), nullable=False, comment='The type of emissions control id: so2, nox, particulate, or mercury.'), + sa.Column('emission_control_id_eia', sa.Text(), nullable=False, comment="The emission control ID used to collect so2, nox, particulate, and mercury emissions data. This column should be used in conjunction with emissions_control_type as it's not guaranteed to be unique."), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__assn_yearly_boiler_emissions_control_equipment_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['core_eia860__scd_boilers.plant_id_eia', 'core_eia860__scd_boilers.boiler_id', 'core_eia860__scd_boilers.report_date'], name=op.f('fk_core_eia860__assn_yearly_boiler_emissions_control_equipment_plant_id_eia_core_eia860__scd_boilers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'boiler_id', 'emission_control_id_type', 'emission_control_id_eia', name=op.f('pk_core_eia860__assn_yearly_boiler_emissions_control_equipment')) ) - op.create_table('capacity_factor_by_generator_yearly', + op.create_table('core_eia860__scd_ownership', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('owner_utility_id_eia', sa.Integer(), nullable=False, comment='The EIA utility Identification number for the owner company that is responsible for the day-to-day operations of the generator, not the operator utility.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_capacity_factor_by_generator_yearly_plant_id_eia_generators_eia860')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_capacity_factor_by_generator_yearly')) + sa.Column('owner_utility_name_eia', sa.Text(), nullable=True, comment='The name of the EIA owner utility.'), + sa.Column('owner_state', sa.Enum('NJ', 'KY', 'PE', 'NT', 'DE', 'MP', 'ON', 'MB', 'OK', 'SD', 'GU', 'CA', 'CT', 'NE', 'AS', 'UT', 'KS', 'MO', 'NC', 'QC', 'NB', 'AL', 'AR', 'BC', 'NU', 'OR', 'VT', 'NM', 'PR', 'WI', 'WA', 'PA', 'RI', 'TX', 'GA', 'MA', 'AB', 'AZ', 'YT', 'ME', 'VI', 'SC', 'IN', 'WV', 'FL', 'NY', 'MS', 'ID', 'DC', 'IL', 'MI', 'CO', 'LA', 'NL', 'MD', 'OH', 'NS', 'AK', 'SK', 'VA', 'MT', 'ND', 'NH', 'MN', 'TN', 'IA', 'HI', 'WY', 'NV'), nullable=True, comment='Two letter ISO-3166 political subdivision code.'), + sa.Column('owner_city', sa.Text(), nullable=True, comment='City of owner.'), + sa.Column('owner_country', sa.Enum('USA', 'CAN'), nullable=True, comment='Three letter ISO-3166 country code.'), + sa.Column('owner_street_address', sa.Text(), nullable=True, comment='Steet address of owner.'), + sa.Column('owner_zip_code', sa.Text(), nullable=True, comment='Zip code of owner.'), + sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_core_eia860__scd_ownership_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['owner_utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_core_eia860__scd_ownership_owner_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_core_eia860__scd_ownership_plant_id_eia_core_eia860__scd_generators')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'owner_utility_id_eia', name=op.f('pk_core_eia860__scd_ownership')) ) - op.create_table('denorm_boiler_fuel_yearly_eia923', + op.create_table('out_eia860__yearly_ownership', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('owner_utility_id_eia', sa.Integer(), nullable=False, comment='The EIA utility Identification number for the owner company that is responsible for the day-to-day operations of the generator, not the operator utility.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('owner_utility_name_eia', sa.Text(), nullable=True, comment='The name of the EIA owner utility.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('owner_state', sa.Enum('NJ', 'KY', 'PE', 'NT', 'DE', 'MP', 'ON', 'MB', 'OK', 'SD', 'GU', 'CA', 'CT', 'NE', 'AS', 'UT', 'KS', 'MO', 'NC', 'QC', 'NB', 'AL', 'AR', 'BC', 'NU', 'OR', 'VT', 'NM', 'PR', 'WI', 'WA', 'PA', 'RI', 'TX', 'GA', 'MA', 'AB', 'AZ', 'YT', 'ME', 'VI', 'SC', 'IN', 'WV', 'FL', 'NY', 'MS', 'ID', 'DC', 'IL', 'MI', 'CO', 'LA', 'NL', 'MD', 'OH', 'NS', 'AK', 'SK', 'VA', 'MT', 'ND', 'NH', 'MN', 'TN', 'IA', 'HI', 'WY', 'NV'), nullable=True, comment='Two letter ISO-3166 political subdivision code.'), + sa.Column('owner_city', sa.Text(), nullable=True, comment='City of owner.'), + sa.Column('owner_country', sa.Enum('USA', 'CAN'), nullable=True, comment='Three letter ISO-3166 country code.'), + sa.Column('owner_street_address', sa.Text(), nullable=True, comment='Steet address of owner.'), + sa.Column('owner_zip_code', sa.Text(), nullable=True, comment='Zip code of owner.'), + sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia860__yearly_ownership_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['owner_utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia860__yearly_ownership_owner_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia860__yearly_ownership_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia860__yearly_ownership_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia860__yearly_ownership_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'owner_utility_id_eia', name=op.f('pk_out_eia860__yearly_ownership')) + ) + op.create_table('out_eia923__yearly_boiler_fuel', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), @@ -3029,17 +3363,91 @@ def upgrade() -> None: sa.Column('fuel_consumed_units', sa.Float(), nullable=True, comment='Consumption of the fuel type in physical unit. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), sa.Column('fuel_mmbtu_per_unit', sa.Float(), nullable=True, comment='Heat content of the fuel in millions of Btus per physical unit.'), sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), - sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_denorm_boiler_fuel_yearly_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['boilers_eia860.plant_id_eia', 'boilers_eia860.boiler_id', 'boilers_eia860.report_date'], name=op.f('fk_denorm_boiler_fuel_yearly_eia923_plant_id_eia_boilers_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_boiler_fuel_yearly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_boiler_fuel_yearly_eia923_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_boiler_fuel_yearly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_denorm_boiler_fuel_yearly_eia923')) - ) - op.create_table('denorm_boilers_eia', + sa.Column('sulfur_content_pct', sa.Float(), nullable=True, comment='Sulfur content percentage by weight to the nearest 0.01 percent.'), + sa.Column('ash_content_pct', sa.Float(), nullable=True, comment='Ash content percentage by weight to the nearest 0.1 percent.'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__yearly_boiler_fuel_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__yearly_boiler_fuel_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['core_eia860__scd_boilers.plant_id_eia', 'core_eia860__scd_boilers.boiler_id', 'core_eia860__scd_boilers.report_date'], name=op.f('fk_out_eia923__yearly_boiler_fuel_plant_id_eia_core_eia860__scd_boilers')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__yearly_boiler_fuel_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__yearly_boiler_fuel_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia923__yearly_boiler_fuel_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__yearly_boiler_fuel_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'energy_source_code', 'prime_mover_code', 'report_date', name=op.f('pk_out_eia923__yearly_boiler_fuel')) + ) + op.create_table('out_eia923__yearly_generation', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia923__yearly_generation_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia923__yearly_generation_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__yearly_generation_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia923__yearly_generation_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__yearly_generation_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_out_eia923__yearly_generation')) + ) + op.create_table('out_eia923__yearly_generation_fuel_by_generator', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_out_eia923__yearly_generation_fuel_by_generator')) + ) + op.create_table('out_eia923__yearly_generation_fuel_by_generator_energy_source', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('energy_source_code_num', sa.Enum('energy_source_code_1', 'energy_source_code_2', 'energy_source_code_3', 'energy_source_code_4', 'energy_source_code_5', 'energy_source_code_6', 'energy_source_code_7', 'energy_source_code_8'), nullable=True, comment='Name of the energy_source_code_N column that this energy source code was reported in for the generator referenced in the same record.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_energy_source_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_energy_source_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_energy_source_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'prime_mover_code', 'energy_source_code', name=op.f('pk_out_eia923__yearly_generation_fuel_by_generator_energy_source')) + ) + op.create_table('out_eia923__yearly_generation_fuel_by_generator_energy_source_owner', + sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), + sa.Column('ownership_record_type', sa.Enum('owned', 'total'), nullable=False, comment='Whether each generator record is for one owner or represents a total of all ownerships.'), + sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('energy_source_code_num', sa.Enum('energy_source_code_1', 'energy_source_code_2', 'energy_source_code_3', 'energy_source_code_4', 'energy_source_code_5', 'energy_source_code_6', 'energy_source_code_7', 'energy_source_code_8'), nullable=True, comment='Name of the energy_source_code_N column that this energy source code was reported in for the generator referenced in the same record.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), + sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), + sa.ForeignKeyConstraint(['energy_source_code'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_energy_source_owner_energy_source_code_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_energy_source_owner_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia923__yearly_generation_fuel_by_generator_energy_source_owner_prime_mover_code_core_eia__codes_prime_movers')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'prime_mover_code', 'energy_source_code', 'utility_id_eia', 'ownership_record_type', name=op.f('pk_out_eia923__yearly_generation_fuel_by_generator_energy_source_owner')) + ) + op.create_table('out_eia__yearly_boilers', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), @@ -3147,93 +3555,93 @@ def upgrade() -> None: sa.Column('waste_heat_input_mmbtu_per_hour', sa.Float(), nullable=True, comment='Design waste-heat input rate at maximum continuous steam flow where a waste-heat boiler is a boiler that receives all or a substantial portion of its energy input from the noncumbustible exhaust gases of a separate fuel-burning process (MMBtu per hour).'), sa.Column('wet_dry_bottom', sa.Text(), nullable=True, comment='Wet or Dry Bottom where Wet Bottom is defined as slag tanks that are installed at furnace throat to contain and remove molten ash from the furnace, and Dry Bottom is defined as having no slag tanks at furnace throat area, throat area is clear, and bottom ash drops through throat to bottom ash water hoppers.'), sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), - sa.ForeignKeyConstraint(['boiler_fuel_code_1'], ['energy_sources_eia.code'], name=op.f('fk_denorm_boilers_eia_boiler_fuel_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_fuel_code_2'], ['energy_sources_eia.code'], name=op.f('fk_denorm_boilers_eia_boiler_fuel_code_2_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_fuel_code_3'], ['energy_sources_eia.code'], name=op.f('fk_denorm_boilers_eia_boiler_fuel_code_3_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_fuel_code_4'], ['energy_sources_eia.code'], name=op.f('fk_denorm_boilers_eia_boiler_fuel_code_4_energy_sources_eia')), - sa.ForeignKeyConstraint(['boiler_manufacturer_code'], ['environmental_equipment_manufacturers_eia.code'], name=op.f('fk_denorm_boilers_eia_boiler_manufacturer_code_environmental_equipment_manufacturers_eia')), - sa.ForeignKeyConstraint(['boiler_status'], ['boiler_status_eia.code'], name=op.f('fk_denorm_boilers_eia_boiler_status_boiler_status_eia')), - sa.ForeignKeyConstraint(['boiler_type'], ['boiler_types_eia.code'], name=op.f('fk_denorm_boilers_eia_boiler_type_boiler_types_eia')), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_boilers_eia_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['firing_type_1'], ['firing_types_eia.code'], name=op.f('fk_denorm_boilers_eia_firing_type_1_firing_types_eia')), - sa.ForeignKeyConstraint(['firing_type_2'], ['firing_types_eia.code'], name=op.f('fk_denorm_boilers_eia_firing_type_2_firing_types_eia')), - sa.ForeignKeyConstraint(['firing_type_3'], ['firing_types_eia.code'], name=op.f('fk_denorm_boilers_eia_firing_type_3_firing_types_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_1'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_existing_strategy_1_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_2'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_existing_strategy_2_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_3'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_existing_strategy_3_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_4'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_existing_strategy_4_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_5'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_existing_strategy_5_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_existing_strategy_6'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_existing_strategy_6_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_1'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_proposed_strategy_1_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_2'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_proposed_strategy_2_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_3'], ['mercury_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_mercury_control_proposed_strategy_3_mercury_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_1'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_existing_caaa_compliance_strategy_1_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_2'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_existing_caaa_compliance_strategy_2_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_3'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_existing_caaa_compliance_strategy_3_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_manufacturer_code'], ['environmental_equipment_manufacturers_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_manufacturer_code_environmental_equipment_manufacturers_eia')), - sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_1'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_out_of_compliance_strategy_1_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_2'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_out_of_compliance_strategy_2_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_3'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_out_of_compliance_strategy_3_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_1'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_planned_caaa_compliance_strategy_1_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_2'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_planned_caaa_compliance_strategy_2_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_3'], ['nox_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_planned_caaa_compliance_strategy_3_nox_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['nox_control_status_code'], ['nox_control_status_eia.code'], name=op.f('fk_denorm_boilers_eia_nox_control_status_code_nox_control_status_eia')), - sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_1'], ['particulate_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_particulate_control_out_of_compliance_strategy_1_particulate_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_2'], ['particulate_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_particulate_control_out_of_compliance_strategy_2_particulate_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_3'], ['particulate_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_particulate_control_out_of_compliance_strategy_3_particulate_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['boilers_eia860.plant_id_eia', 'boilers_eia860.boiler_id', 'boilers_eia860.report_date'], name=op.f('fk_denorm_boilers_eia_plant_id_eia_boilers_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_boilers_eia_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['regulation_mercury'], ['regulations_eia.code'], name=op.f('fk_denorm_boilers_eia_regulation_mercury_regulations_eia')), - sa.ForeignKeyConstraint(['regulation_nox'], ['regulations_eia.code'], name=op.f('fk_denorm_boilers_eia_regulation_nox_regulations_eia')), - sa.ForeignKeyConstraint(['regulation_particulate'], ['regulations_eia.code'], name=op.f('fk_denorm_boilers_eia_regulation_particulate_regulations_eia')), - sa.ForeignKeyConstraint(['regulation_so2'], ['regulations_eia.code'], name=op.f('fk_denorm_boilers_eia_regulation_so2_regulations_eia')), - sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_1'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_existing_caaa_compliance_strategy_1_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_2'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_existing_caaa_compliance_strategy_2_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_3'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_existing_caaa_compliance_strategy_3_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_1'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_out_of_compliance_strategy_1_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_2'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_out_of_compliance_strategy_2_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_3'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_out_of_compliance_strategy_3_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_1'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_planned_caaa_compliance_strategy_1_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_2'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_planned_caaa_compliance_strategy_2_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_3'], ['so2_compliance_strategies_eia.code'], name=op.f('fk_denorm_boilers_eia_so2_control_planned_caaa_compliance_strategy_3_so2_compliance_strategies_eia')), - sa.ForeignKeyConstraint(['unit_nox'], ['nox_units_eia.code'], name=op.f('fk_denorm_boilers_eia_unit_nox_nox_units_eia')), - sa.ForeignKeyConstraint(['unit_particulate'], ['particulate_units_eia.code'], name=op.f('fk_denorm_boilers_eia_unit_particulate_particulate_units_eia')), - sa.ForeignKeyConstraint(['unit_so2'], ['so2_units_eia.code'], name=op.f('fk_denorm_boilers_eia_unit_so2_so2_units_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_boilers_eia_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_boilers_eia_utility_id_pudl_utilities_pudl')), - sa.ForeignKeyConstraint(['wet_dry_bottom'], ['wet_dry_bottom_eia.code'], name=op.f('fk_denorm_boilers_eia_wet_dry_bottom_wet_dry_bottom_eia')), - sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'report_date', name=op.f('pk_denorm_boilers_eia')) - ) - op.create_table('denorm_generation_yearly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), + sa.ForeignKeyConstraint(['boiler_fuel_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_boilers_boiler_fuel_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_fuel_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_boilers_boiler_fuel_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_fuel_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_boilers_boiler_fuel_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_fuel_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_boilers_boiler_fuel_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['boiler_manufacturer_code'], ['core_eia__codes_environmental_equipment_manufacturers.code'], name=op.f('fk_out_eia__yearly_boilers_boiler_manufacturer_code_core_eia__codes_environmental_equipment_manufacturers')), + sa.ForeignKeyConstraint(['boiler_status'], ['core_eia__codes_boiler_status.code'], name=op.f('fk_out_eia__yearly_boilers_boiler_status_core_eia__codes_boiler_status')), + sa.ForeignKeyConstraint(['boiler_type'], ['core_eia__codes_boiler_types.code'], name=op.f('fk_out_eia__yearly_boilers_boiler_type_core_eia__codes_boiler_types')), + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia__yearly_boilers_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['firing_type_1'], ['core_eia__codes_firing_types.code'], name=op.f('fk_out_eia__yearly_boilers_firing_type_1_core_eia__codes_firing_types')), + sa.ForeignKeyConstraint(['firing_type_2'], ['core_eia__codes_firing_types.code'], name=op.f('fk_out_eia__yearly_boilers_firing_type_2_core_eia__codes_firing_types')), + sa.ForeignKeyConstraint(['firing_type_3'], ['core_eia__codes_firing_types.code'], name=op.f('fk_out_eia__yearly_boilers_firing_type_3_core_eia__codes_firing_types')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_1'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_existing_strategy_1_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_2'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_existing_strategy_2_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_3'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_existing_strategy_3_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_4'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_existing_strategy_4_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_5'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_existing_strategy_5_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_existing_strategy_6'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_existing_strategy_6_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_1'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_proposed_strategy_1_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_2'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_proposed_strategy_2_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['mercury_control_proposed_strategy_3'], ['core_eia__codes_mercury_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_mercury_control_proposed_strategy_3_core_eia__codes_mercury_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_1'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_existing_caaa_compliance_strategy_1_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_2'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_existing_caaa_compliance_strategy_2_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_existing_caaa_compliance_strategy_3'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_existing_caaa_compliance_strategy_3_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_manufacturer_code'], ['core_eia__codes_environmental_equipment_manufacturers.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_manufacturer_code_core_eia__codes_environmental_equipment_manufacturers')), + sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_1'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_out_of_compliance_strategy_1_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_2'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_out_of_compliance_strategy_2_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_out_of_compliance_strategy_3'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_out_of_compliance_strategy_3_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_1'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_planned_caaa_compliance_strategy_1_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_2'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_planned_caaa_compliance_strategy_2_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_planned_caaa_compliance_strategy_3'], ['core_eia__codes_nox_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_planned_caaa_compliance_strategy_3_core_eia__codes_nox_compliance_strategies')), + sa.ForeignKeyConstraint(['nox_control_status_code'], ['core_eia__codes_nox_control_status.code'], name=op.f('fk_out_eia__yearly_boilers_nox_control_status_code_core_eia__codes_nox_control_status')), + sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_1'], ['core_eia__codes_particulate_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_particulate_control_out_of_compliance_strategy_1_core_eia__codes_particulate_compliance_strategies')), + sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_2'], ['core_eia__codes_particulate_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_particulate_control_out_of_compliance_strategy_2_core_eia__codes_particulate_compliance_strategies')), + sa.ForeignKeyConstraint(['particulate_control_out_of_compliance_strategy_3'], ['core_eia__codes_particulate_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_particulate_control_out_of_compliance_strategy_3_core_eia__codes_particulate_compliance_strategies')), + sa.ForeignKeyConstraint(['plant_id_eia', 'boiler_id', 'report_date'], ['core_eia860__scd_boilers.plant_id_eia', 'core_eia860__scd_boilers.boiler_id', 'core_eia860__scd_boilers.report_date'], name=op.f('fk_out_eia__yearly_boilers_plant_id_eia_core_eia860__scd_boilers')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia__yearly_boilers_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['regulation_mercury'], ['core_eia__codes_regulations.code'], name=op.f('fk_out_eia__yearly_boilers_regulation_mercury_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['regulation_nox'], ['core_eia__codes_regulations.code'], name=op.f('fk_out_eia__yearly_boilers_regulation_nox_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['regulation_particulate'], ['core_eia__codes_regulations.code'], name=op.f('fk_out_eia__yearly_boilers_regulation_particulate_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['regulation_so2'], ['core_eia__codes_regulations.code'], name=op.f('fk_out_eia__yearly_boilers_regulation_so2_core_eia__codes_regulations')), + sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_1'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_existing_caaa_compliance_strategy_1_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_2'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_existing_caaa_compliance_strategy_2_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_existing_caaa_compliance_strategy_3'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_existing_caaa_compliance_strategy_3_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_1'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_out_of_compliance_strategy_1_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_2'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_out_of_compliance_strategy_2_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_out_of_compliance_strategy_3'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_out_of_compliance_strategy_3_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_1'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_planned_caaa_compliance_strategy_1_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_2'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_planned_caaa_compliance_strategy_2_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['so2_control_planned_caaa_compliance_strategy_3'], ['core_eia__codes_so2_compliance_strategies.code'], name=op.f('fk_out_eia__yearly_boilers_so2_control_planned_caaa_compliance_strategy_3_core_eia__codes_so2_compliance_strategies')), + sa.ForeignKeyConstraint(['unit_nox'], ['core_eia__codes_nox_units.code'], name=op.f('fk_out_eia__yearly_boilers_unit_nox_core_eia__codes_nox_units')), + sa.ForeignKeyConstraint(['unit_particulate'], ['core_eia__codes_particulate_units.code'], name=op.f('fk_out_eia__yearly_boilers_unit_particulate_core_eia__codes_particulate_units')), + sa.ForeignKeyConstraint(['unit_so2'], ['core_eia__codes_so2_units.code'], name=op.f('fk_out_eia__yearly_boilers_unit_so2_core_eia__codes_so2_units')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia__yearly_boilers_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia__yearly_boilers_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.ForeignKeyConstraint(['wet_dry_bottom'], ['core_eia__codes_wet_dry_bottom.code'], name=op.f('fk_out_eia__yearly_boilers_wet_dry_bottom_core_eia__codes_wet_dry_bottom')), + sa.PrimaryKeyConstraint('plant_id_eia', 'boiler_id', 'report_date', name=op.f('pk_out_eia__yearly_boilers')) + ) + op.create_table('out_eia__yearly_generators', sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_denorm_generation_yearly_eia923_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_generation_yearly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_generation_yearly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_denorm_generation_yearly_eia923')) - ) - op.create_table('denorm_generators_eia', sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), + sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), + sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), + sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), + sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), + sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), + sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), + sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), sa.Column('associated_combined_heat_power', sa.Boolean(), nullable=True, comment='Indicates whether the generator is associated with a combined heat and power system'), sa.Column('bga_source', sa.Text(), nullable=True, comment='The source from where the unit_id_pudl is compiled. The unit_id_pudl comes directly from EIA 860, or string association (which looks at all the boilers and generators that are not associated with a unit and tries to find a matching string in the respective collection of boilers or generator), or from a unit connection (where the unit_id_eia is employed to find additional boiler generator connections).'), sa.Column('bypass_heat_recovery', sa.Boolean(), nullable=True, comment='Can this generator operate while bypassing the heat recovery steam generator?'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), sa.Column('carbon_capture', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses carbon capture technology.'), sa.Column('city', sa.Text(), nullable=True), sa.Column('cofire_fuels', sa.Boolean(), nullable=True, comment='Can the generator co-fire fuels?.'), @@ -3249,7 +3657,6 @@ def upgrade() -> None: sa.Column('energy_source_2_transport_1', sa.Text(), nullable=True, comment='Primary mode of transport for energy source 2.'), sa.Column('energy_source_2_transport_2', sa.Text(), nullable=True, comment='Secondary mode of transport for energy source 2.'), sa.Column('energy_source_2_transport_3', sa.Text(), nullable=True, comment='Tertiary mode of transport for energy source 2.'), - sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), sa.Column('energy_source_code_2', sa.Text(), nullable=True, comment='The code representing the second most predominant type of energy that fuels the generator'), sa.Column('energy_source_code_3', sa.Text(), nullable=True, comment='The code representing the third most predominant type of energy that fuels the generator'), sa.Column('energy_source_code_4', sa.Text(), nullable=True, comment='The code representing the fourth most predominant type of energy that fuels the generator'), @@ -3258,10 +3665,7 @@ def upgrade() -> None: sa.Column('energy_storage_capacity_mwh', sa.Float(), nullable=True, comment='Energy storage capacity in MWh (e.g. for batteries).'), sa.Column('ferc_qualifying_facility', sa.Boolean(), nullable=True, comment='Indicatates whether or not a generator is a qualifying FERC cogeneation facility.'), sa.Column('fluidized_bed_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses fluidized bed technology'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), - sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), - sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), sa.Column('latitude', sa.Float(), nullable=True, comment="Latitude of the plant's location, in degrees."), sa.Column('longitude', sa.Float(), nullable=True, comment="Longitude of the plant's location, in degrees."), sa.Column('minimum_load_mw', sa.Float(), nullable=True, comment='The minimum load at which the generator can operate at continuosuly.'), @@ -3269,7 +3673,6 @@ def upgrade() -> None: sa.Column('nameplate_power_factor', sa.Float(), nullable=True, comment='The nameplate power factor of the generator.'), sa.Column('net_capacity_mwdc', sa.Float(), nullable=True, comment='Generation capacity in megawatts of direct current that is subject to a net metering agreement. Typically used for behind-the-meter solar PV.'), sa.Column('operating_switch', sa.Text(), nullable=True, comment='Indicates whether the fuel switching generator can switch when operating'), - sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), sa.Column('operational_status_code', sa.Text(), nullable=True, comment='The operating status of the asset.'), sa.Column('original_planned_generator_operating_date', sa.Date(), nullable=True, comment='The date the generator was originally scheduled to be operational'), sa.Column('other_combustion_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses other combustion technologies'), @@ -3279,7 +3682,6 @@ def upgrade() -> None: sa.Column('ownership_code', sa.Text(), nullable=True, comment='Identifies the ownership for each generator.'), sa.Column('planned_derate_date', sa.Date(), nullable=True, comment='Planned effective month that the generator is scheduled to enter operation after the derate modification.'), sa.Column('planned_energy_source_code_1', sa.Text(), nullable=True, comment='New energy source code for the planned repowered generator.'), - sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), sa.Column('planned_modifications', sa.Boolean(), nullable=True, comment='Indicates whether there are any planned capacity uprates/derates, repowering, other modifications, or generator retirements scheduled for the next 5 years.'), sa.Column('planned_net_summer_capacity_derate_mw', sa.Float(), nullable=True, comment='Decrease in summer capacity expected to be realized from the derate modification to the equipment.'), sa.Column('planned_net_summer_capacity_uprate_mw', sa.Float(), nullable=True, comment='Increase in summer capacity expected to be realized from the modification to the equipment.'), @@ -3290,7 +3692,6 @@ def upgrade() -> None: sa.Column('planned_repower_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the repowering is complete.'), sa.Column('planned_uprate_date', sa.Date(), nullable=True, comment='Planned effective date that the generator is scheduled to enter operation after the uprate modification.'), sa.Column('previously_canceled', sa.Boolean(), nullable=True, comment='Indicates whether the generator was previously reported as indefinitely postponed or canceled'), - sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), sa.Column('pulverized_coal_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses pulverized coal technology'), sa.Column('reactive_power_output_mvar', sa.Float(), nullable=True, comment='Reactive Power Output (MVAr)'), sa.Column('rto_iso_lmp_node_id', sa.Text(), nullable=True, comment='The designation used to identify the price node in RTO/ISO Locational Marginal Price reports'), @@ -3310,209 +3711,46 @@ def upgrade() -> None: sa.Column('supercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses supercritical technology'), sa.Column('switch_oil_gas', sa.Boolean(), nullable=True, comment='Indicates whether the generator switch between oil and natural gas.'), sa.Column('syncronized_transmission_grid', sa.Boolean(), nullable=True, comment='Indicates whether standby generators (SB status) can be synchronized to the grid.'), - sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), sa.Column('time_cold_shutdown_full_load_code', sa.Text(), nullable=True, comment='The minimum amount of time required to bring the unit to full load from shutdown.'), sa.Column('timezone', sa.Enum('Africa/Abidjan', 'Africa/Accra', 'Africa/Addis_Ababa', 'Africa/Algiers', 'Africa/Asmara', 'Africa/Asmera', 'Africa/Bamako', 'Africa/Bangui', 'Africa/Banjul', 'Africa/Bissau', 'Africa/Blantyre', 'Africa/Brazzaville', 'Africa/Bujumbura', 'Africa/Cairo', 'Africa/Casablanca', 'Africa/Ceuta', 'Africa/Conakry', 'Africa/Dakar', 'Africa/Dar_es_Salaam', 'Africa/Djibouti', 'Africa/Douala', 'Africa/El_Aaiun', 'Africa/Freetown', 'Africa/Gaborone', 'Africa/Harare', 'Africa/Johannesburg', 'Africa/Juba', 'Africa/Kampala', 'Africa/Khartoum', 'Africa/Kigali', 'Africa/Kinshasa', 'Africa/Lagos', 'Africa/Libreville', 'Africa/Lome', 'Africa/Luanda', 'Africa/Lubumbashi', 'Africa/Lusaka', 'Africa/Malabo', 'Africa/Maputo', 'Africa/Maseru', 'Africa/Mbabane', 'Africa/Mogadishu', 'Africa/Monrovia', 'Africa/Nairobi', 'Africa/Ndjamena', 'Africa/Niamey', 'Africa/Nouakchott', 'Africa/Ouagadougou', 'Africa/Porto-Novo', 'Africa/Sao_Tome', 'Africa/Timbuktu', 'Africa/Tripoli', 'Africa/Tunis', 'Africa/Windhoek', 'America/Adak', 'America/Anchorage', 'America/Anguilla', 'America/Antigua', 'America/Araguaina', 'America/Argentina/Buenos_Aires', 'America/Argentina/Catamarca', 'America/Argentina/ComodRivadavia', 'America/Argentina/Cordoba', 'America/Argentina/Jujuy', 'America/Argentina/La_Rioja', 'America/Argentina/Mendoza', 'America/Argentina/Rio_Gallegos', 'America/Argentina/Salta', 'America/Argentina/San_Juan', 'America/Argentina/San_Luis', 'America/Argentina/Tucuman', 'America/Argentina/Ushuaia', 'America/Aruba', 'America/Asuncion', 'America/Atikokan', 'America/Atka', 'America/Bahia', 'America/Bahia_Banderas', 'America/Barbados', 'America/Belem', 'America/Belize', 'America/Blanc-Sablon', 'America/Boa_Vista', 'America/Bogota', 'America/Boise', 'America/Buenos_Aires', 'America/Cambridge_Bay', 'America/Campo_Grande', 'America/Cancun', 'America/Caracas', 'America/Catamarca', 'America/Cayenne', 'America/Cayman', 'America/Chicago', 'America/Chihuahua', 'America/Ciudad_Juarez', 'America/Coral_Harbour', 'America/Cordoba', 'America/Costa_Rica', 'America/Creston', 'America/Cuiaba', 'America/Curacao', 'America/Danmarkshavn', 'America/Dawson', 'America/Dawson_Creek', 'America/Denver', 'America/Detroit', 'America/Dominica', 'America/Edmonton', 'America/Eirunepe', 'America/El_Salvador', 'America/Ensenada', 'America/Fort_Nelson', 'America/Fort_Wayne', 'America/Fortaleza', 'America/Glace_Bay', 'America/Godthab', 'America/Goose_Bay', 'America/Grand_Turk', 'America/Grenada', 'America/Guadeloupe', 'America/Guatemala', 'America/Guayaquil', 'America/Guyana', 'America/Halifax', 'America/Havana', 'America/Hermosillo', 'America/Indiana/Indianapolis', 'America/Indiana/Knox', 'America/Indiana/Marengo', 'America/Indiana/Petersburg', 'America/Indiana/Tell_City', 'America/Indiana/Vevay', 'America/Indiana/Vincennes', 'America/Indiana/Winamac', 'America/Indianapolis', 'America/Inuvik', 'America/Iqaluit', 'America/Jamaica', 'America/Jujuy', 'America/Juneau', 'America/Kentucky/Louisville', 'America/Kentucky/Monticello', 'America/Knox_IN', 'America/Kralendijk', 'America/La_Paz', 'America/Lima', 'America/Los_Angeles', 'America/Louisville', 'America/Lower_Princes', 'America/Maceio', 'America/Managua', 'America/Manaus', 'America/Marigot', 'America/Martinique', 'America/Matamoros', 'America/Mazatlan', 'America/Mendoza', 'America/Menominee', 'America/Merida', 'America/Metlakatla', 'America/Mexico_City', 'America/Miquelon', 'America/Moncton', 'America/Monterrey', 'America/Montevideo', 'America/Montreal', 'America/Montserrat', 'America/Nassau', 'America/New_York', 'America/Nipigon', 'America/Nome', 'America/Noronha', 'America/North_Dakota/Beulah', 'America/North_Dakota/Center', 'America/North_Dakota/New_Salem', 'America/Nuuk', 'America/Ojinaga', 'America/Panama', 'America/Pangnirtung', 'America/Paramaribo', 'America/Phoenix', 'America/Port-au-Prince', 'America/Port_of_Spain', 'America/Porto_Acre', 'America/Porto_Velho', 'America/Puerto_Rico', 'America/Punta_Arenas', 'America/Rainy_River', 'America/Rankin_Inlet', 'America/Recife', 'America/Regina', 'America/Resolute', 'America/Rio_Branco', 'America/Rosario', 'America/Santa_Isabel', 'America/Santarem', 'America/Santiago', 'America/Santo_Domingo', 'America/Sao_Paulo', 'America/Scoresbysund', 'America/Shiprock', 'America/Sitka', 'America/St_Barthelemy', 'America/St_Johns', 'America/St_Kitts', 'America/St_Lucia', 'America/St_Thomas', 'America/St_Vincent', 'America/Swift_Current', 'America/Tegucigalpa', 'America/Thule', 'America/Thunder_Bay', 'America/Tijuana', 'America/Toronto', 'America/Tortola', 'America/Vancouver', 'America/Virgin', 'America/Whitehorse', 'America/Winnipeg', 'America/Yakutat', 'America/Yellowknife', 'Antarctica/Casey', 'Antarctica/Davis', 'Antarctica/DumontDUrville', 'Antarctica/Macquarie', 'Antarctica/Mawson', 'Antarctica/McMurdo', 'Antarctica/Palmer', 'Antarctica/Rothera', 'Antarctica/South_Pole', 'Antarctica/Syowa', 'Antarctica/Troll', 'Antarctica/Vostok', 'Arctic/Longyearbyen', 'Asia/Aden', 'Asia/Almaty', 'Asia/Amman', 'Asia/Anadyr', 'Asia/Aqtau', 'Asia/Aqtobe', 'Asia/Ashgabat', 'Asia/Ashkhabad', 'Asia/Atyrau', 'Asia/Baghdad', 'Asia/Bahrain', 'Asia/Baku', 'Asia/Bangkok', 'Asia/Barnaul', 'Asia/Beirut', 'Asia/Bishkek', 'Asia/Brunei', 'Asia/Calcutta', 'Asia/Chita', 'Asia/Choibalsan', 'Asia/Chongqing', 'Asia/Chungking', 'Asia/Colombo', 'Asia/Dacca', 'Asia/Damascus', 'Asia/Dhaka', 'Asia/Dili', 'Asia/Dubai', 'Asia/Dushanbe', 'Asia/Famagusta', 'Asia/Gaza', 'Asia/Harbin', 'Asia/Hebron', 'Asia/Ho_Chi_Minh', 'Asia/Hong_Kong', 'Asia/Hovd', 'Asia/Irkutsk', 'Asia/Istanbul', 'Asia/Jakarta', 'Asia/Jayapura', 'Asia/Jerusalem', 'Asia/Kabul', 'Asia/Kamchatka', 'Asia/Karachi', 'Asia/Kashgar', 'Asia/Kathmandu', 'Asia/Katmandu', 'Asia/Khandyga', 'Asia/Kolkata', 'Asia/Krasnoyarsk', 'Asia/Kuala_Lumpur', 'Asia/Kuching', 'Asia/Kuwait', 'Asia/Macao', 'Asia/Macau', 'Asia/Magadan', 'Asia/Makassar', 'Asia/Manila', 'Asia/Muscat', 'Asia/Nicosia', 'Asia/Novokuznetsk', 'Asia/Novosibirsk', 'Asia/Omsk', 'Asia/Oral', 'Asia/Phnom_Penh', 'Asia/Pontianak', 'Asia/Pyongyang', 'Asia/Qatar', 'Asia/Qostanay', 'Asia/Qyzylorda', 'Asia/Rangoon', 'Asia/Riyadh', 'Asia/Saigon', 'Asia/Sakhalin', 'Asia/Samarkand', 'Asia/Seoul', 'Asia/Shanghai', 'Asia/Singapore', 'Asia/Srednekolymsk', 'Asia/Taipei', 'Asia/Tashkent', 'Asia/Tbilisi', 'Asia/Tehran', 'Asia/Tel_Aviv', 'Asia/Thimbu', 'Asia/Thimphu', 'Asia/Tokyo', 'Asia/Tomsk', 'Asia/Ujung_Pandang', 'Asia/Ulaanbaatar', 'Asia/Ulan_Bator', 'Asia/Urumqi', 'Asia/Ust-Nera', 'Asia/Vientiane', 'Asia/Vladivostok', 'Asia/Yakutsk', 'Asia/Yangon', 'Asia/Yekaterinburg', 'Asia/Yerevan', 'Atlantic/Azores', 'Atlantic/Bermuda', 'Atlantic/Canary', 'Atlantic/Cape_Verde', 'Atlantic/Faeroe', 'Atlantic/Faroe', 'Atlantic/Jan_Mayen', 'Atlantic/Madeira', 'Atlantic/Reykjavik', 'Atlantic/South_Georgia', 'Atlantic/St_Helena', 'Atlantic/Stanley', 'Australia/ACT', 'Australia/Adelaide', 'Australia/Brisbane', 'Australia/Broken_Hill', 'Australia/Canberra', 'Australia/Currie', 'Australia/Darwin', 'Australia/Eucla', 'Australia/Hobart', 'Australia/LHI', 'Australia/Lindeman', 'Australia/Lord_Howe', 'Australia/Melbourne', 'Australia/NSW', 'Australia/North', 'Australia/Perth', 'Australia/Queensland', 'Australia/South', 'Australia/Sydney', 'Australia/Tasmania', 'Australia/Victoria', 'Australia/West', 'Australia/Yancowinna', 'Brazil/Acre', 'Brazil/DeNoronha', 'Brazil/East', 'Brazil/West', 'CET', 'CST6CDT', 'Canada/Atlantic', 'Canada/Central', 'Canada/Eastern', 'Canada/Mountain', 'Canada/Newfoundland', 'Canada/Pacific', 'Canada/Saskatchewan', 'Canada/Yukon', 'Chile/Continental', 'Chile/EasterIsland', 'Cuba', 'EET', 'EST', 'EST5EDT', 'Egypt', 'Eire', 'Etc/GMT', 'Etc/GMT+0', 'Etc/GMT+1', 'Etc/GMT+10', 'Etc/GMT+11', 'Etc/GMT+12', 'Etc/GMT+2', 'Etc/GMT+3', 'Etc/GMT+4', 'Etc/GMT+5', 'Etc/GMT+6', 'Etc/GMT+7', 'Etc/GMT+8', 'Etc/GMT+9', 'Etc/GMT-0', 'Etc/GMT-1', 'Etc/GMT-10', 'Etc/GMT-11', 'Etc/GMT-12', 'Etc/GMT-13', 'Etc/GMT-14', 'Etc/GMT-2', 'Etc/GMT-3', 'Etc/GMT-4', 'Etc/GMT-5', 'Etc/GMT-6', 'Etc/GMT-7', 'Etc/GMT-8', 'Etc/GMT-9', 'Etc/GMT0', 'Etc/Greenwich', 'Etc/UCT', 'Etc/UTC', 'Etc/Universal', 'Etc/Zulu', 'Europe/Amsterdam', 'Europe/Andorra', 'Europe/Astrakhan', 'Europe/Athens', 'Europe/Belfast', 'Europe/Belgrade', 'Europe/Berlin', 'Europe/Bratislava', 'Europe/Brussels', 'Europe/Bucharest', 'Europe/Budapest', 'Europe/Busingen', 'Europe/Chisinau', 'Europe/Copenhagen', 'Europe/Dublin', 'Europe/Gibraltar', 'Europe/Guernsey', 'Europe/Helsinki', 'Europe/Isle_of_Man', 'Europe/Istanbul', 'Europe/Jersey', 'Europe/Kaliningrad', 'Europe/Kiev', 'Europe/Kirov', 'Europe/Kyiv', 'Europe/Lisbon', 'Europe/Ljubljana', 'Europe/London', 'Europe/Luxembourg', 'Europe/Madrid', 'Europe/Malta', 'Europe/Mariehamn', 'Europe/Minsk', 'Europe/Monaco', 'Europe/Moscow', 'Europe/Nicosia', 'Europe/Oslo', 'Europe/Paris', 'Europe/Podgorica', 'Europe/Prague', 'Europe/Riga', 'Europe/Rome', 'Europe/Samara', 'Europe/San_Marino', 'Europe/Sarajevo', 'Europe/Saratov', 'Europe/Simferopol', 'Europe/Skopje', 'Europe/Sofia', 'Europe/Stockholm', 'Europe/Tallinn', 'Europe/Tirane', 'Europe/Tiraspol', 'Europe/Ulyanovsk', 'Europe/Uzhgorod', 'Europe/Vaduz', 'Europe/Vatican', 'Europe/Vienna', 'Europe/Vilnius', 'Europe/Volgograd', 'Europe/Warsaw', 'Europe/Zagreb', 'Europe/Zaporozhye', 'Europe/Zurich', 'GB', 'GB-Eire', 'GMT', 'GMT+0', 'GMT-0', 'GMT0', 'Greenwich', 'HST', 'Hongkong', 'Iceland', 'Indian/Antananarivo', 'Indian/Chagos', 'Indian/Christmas', 'Indian/Cocos', 'Indian/Comoro', 'Indian/Kerguelen', 'Indian/Mahe', 'Indian/Maldives', 'Indian/Mauritius', 'Indian/Mayotte', 'Indian/Reunion', 'Iran', 'Israel', 'Jamaica', 'Japan', 'Kwajalein', 'Libya', 'MET', 'MST', 'MST7MDT', 'Mexico/BajaNorte', 'Mexico/BajaSur', 'Mexico/General', 'NZ', 'NZ-CHAT', 'Navajo', 'PRC', 'PST8PDT', 'Pacific/Apia', 'Pacific/Auckland', 'Pacific/Bougainville', 'Pacific/Chatham', 'Pacific/Chuuk', 'Pacific/Easter', 'Pacific/Efate', 'Pacific/Enderbury', 'Pacific/Fakaofo', 'Pacific/Fiji', 'Pacific/Funafuti', 'Pacific/Galapagos', 'Pacific/Gambier', 'Pacific/Guadalcanal', 'Pacific/Guam', 'Pacific/Honolulu', 'Pacific/Johnston', 'Pacific/Kanton', 'Pacific/Kiritimati', 'Pacific/Kosrae', 'Pacific/Kwajalein', 'Pacific/Majuro', 'Pacific/Marquesas', 'Pacific/Midway', 'Pacific/Nauru', 'Pacific/Niue', 'Pacific/Norfolk', 'Pacific/Noumea', 'Pacific/Pago_Pago', 'Pacific/Palau', 'Pacific/Pitcairn', 'Pacific/Pohnpei', 'Pacific/Ponape', 'Pacific/Port_Moresby', 'Pacific/Rarotonga', 'Pacific/Saipan', 'Pacific/Samoa', 'Pacific/Tahiti', 'Pacific/Tarawa', 'Pacific/Tongatapu', 'Pacific/Truk', 'Pacific/Wake', 'Pacific/Wallis', 'Pacific/Yap', 'Poland', 'Portugal', 'ROC', 'ROK', 'Singapore', 'Turkey', 'UCT', 'US/Alaska', 'US/Aleutian', 'US/Arizona', 'US/Central', 'US/East-Indiana', 'US/Eastern', 'US/Hawaii', 'US/Indiana-Starke', 'US/Michigan', 'US/Mountain', 'US/Pacific', 'US/Samoa', 'UTC', 'Universal', 'W-SU', 'WET', 'Zulu'), nullable=True, comment='IANA timezone name'), sa.Column('topping_bottoming_code', sa.Text(), nullable=True, comment='If the generator is associated with a combined heat and power system, indicates whether the generator is part of a topping cycle or a bottoming cycle'), sa.Column('turbines_inverters_hydrokinetics', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), sa.Column('turbines_num', sa.Integer(), nullable=True, comment='Number of wind turbines, or hydrokinetic buoys.'), sa.Column('ultrasupercritical_tech', sa.Boolean(), nullable=True, comment='Indicates whether the generator uses ultra-supercritical technology'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), sa.Column('uprate_derate_completed_date', sa.Date(), nullable=True, comment='The date when the uprate or derate was completed.'), sa.Column('uprate_derate_during_year', sa.Boolean(), nullable=True, comment='Was an uprate or derate completed on this generator during the reporting year?'), sa.Column('winter_capacity_estimate', sa.Boolean(), nullable=True, comment='Whether the winter capacity value was an estimate'), sa.Column('winter_capacity_mw', sa.Float(), nullable=True, comment='The net winter capacity.'), sa.Column('winter_estimated_capability_mw', sa.Float(), nullable=True, comment='EIA estimated winter capacity (in MWh).'), sa.Column('zip_code', sa.Text(), nullable=True, comment='Five digit US Zip Code.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_generators_eia_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['energy_source_1_transport_1'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_1_transport_1_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_1_transport_2'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_1_transport_2_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_1_transport_3'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_1_transport_3_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_2_transport_1'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_2_transport_1_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_2_transport_2'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_2_transport_2_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_2_transport_3'], ['fuel_transportation_modes_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_2_transport_3_fuel_transportation_modes_eia')), - sa.ForeignKeyConstraint(['energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_2'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_code_2_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_3'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_code_3_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_4'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_code_4_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_5'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_code_5_energy_sources_eia')), - sa.ForeignKeyConstraint(['energy_source_code_6'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_energy_source_code_6_energy_sources_eia')), - sa.ForeignKeyConstraint(['operational_status_code'], ['operational_status_eia.code'], name=op.f('fk_denorm_generators_eia_operational_status_code_operational_status_eia')), - sa.ForeignKeyConstraint(['planned_energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_planned_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['planned_new_prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_generators_eia_planned_new_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_denorm_generators_eia_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_generators_eia_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_denorm_generators_eia_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['startup_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_startup_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['startup_source_code_2'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_startup_source_code_2_energy_sources_eia')), - sa.ForeignKeyConstraint(['startup_source_code_3'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_startup_source_code_3_energy_sources_eia')), - sa.ForeignKeyConstraint(['startup_source_code_4'], ['energy_sources_eia.code'], name=op.f('fk_denorm_generators_eia_startup_source_code_4_energy_sources_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_denorm_generators_eia_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_generators_eia_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('plant_id_eia', 'generator_id', 'report_date', name=op.f('pk_denorm_generators_eia')) - ) - op.create_table('denorm_ownership_eia860', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('owner_utility_id_eia', sa.Integer(), nullable=False, comment='The EIA utility Identification number for the owner company that is responsible for the day-to-day operations of the generator, not the operator utility.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('owner_utility_name_eia', sa.Text(), nullable=True, comment='The name of the EIA owner utility.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('state', sa.Text(), nullable=True, comment='Two letter US state abbreviation.'), - sa.Column('owner_city', sa.Text(), nullable=True, comment='City of owner.'), - sa.Column('owner_country', sa.Enum('USA', 'CAN'), nullable=True, comment='Three letter ISO-3166 country code.'), - sa.Column('owner_street_address', sa.Text(), nullable=True, comment='Steet address of owner.'), - sa.Column('owner_zip_code', sa.Text(), nullable=True, comment='Zip code of owner.'), - sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), - sa.Column('data_maturity', sa.Text(), nullable=True, comment='Level of maturity of the data record. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.'), - sa.ForeignKeyConstraint(['data_maturity'], ['data_maturities.code'], name=op.f('fk_denorm_ownership_eia860_data_maturity_data_maturities')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_denorm_ownership_eia860_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_denorm_ownership_eia860_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_denorm_ownership_eia860_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'owner_utility_id_eia', name=op.f('pk_denorm_ownership_eia860')) - ) - op.create_table('fuel_cost_by_generator_yearly', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_fuel_cost_by_generator_yearly_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_fuel_cost_by_generator_yearly_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_fuel_cost_by_generator_yearly_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_fuel_cost_by_generator_yearly')) - ) - op.create_table('generation_fuel_by_generator_energy_source_owner_yearly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=False, comment='The EIA Utility Identification number.'), - sa.Column('ownership_record_type', sa.Enum('owned', 'total'), nullable=False, comment='Whether each generator record is for one owner or represents a total of all ownerships.'), - sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('energy_source_code_num', sa.Enum('energy_source_code_1', 'energy_source_code_2', 'energy_source_code_3', 'energy_source_code_4', 'energy_source_code_5', 'energy_source_code_6', 'energy_source_code_7', 'energy_source_code_8'), nullable=True, comment='Name of the energy_source_code_N column that this energy source code was reported in for the generator referenced in the same record.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_generation_fuel_by_generator_energy_source_owner_yearly_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_generation_fuel_by_generator_energy_source_owner_yearly_eia923_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_generation_fuel_by_generator_energy_source_owner_yearly_eia923_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'prime_mover_code', 'energy_source_code', 'utility_id_eia', 'ownership_record_type', name=op.f('pk_generation_fuel_by_generator_energy_source_owner_yearly_eia923')) - ) - op.create_table('generation_fuel_by_generator_energy_source_yearly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('prime_mover_code', sa.Text(), nullable=False, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('energy_source_code', sa.Text(), nullable=False, comment='A 2-3 letter code indicating the energy source (e.g. fuel type) associated with the record.'), - sa.Column('energy_source_code_num', sa.Enum('energy_source_code_1', 'energy_source_code_2', 'energy_source_code_3', 'energy_source_code_4', 'energy_source_code_5', 'energy_source_code_6', 'energy_source_code_7', 'energy_source_code_8'), nullable=True, comment='Name of the energy_source_code_N column that this energy source code was reported in for the generator referenced in the same record.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.ForeignKeyConstraint(['energy_source_code'], ['energy_sources_eia.code'], name=op.f('fk_generation_fuel_by_generator_energy_source_yearly_eia923_energy_source_code_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_generation_fuel_by_generator_energy_source_yearly_eia923_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_generation_fuel_by_generator_energy_source_yearly_eia923_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', 'prime_mover_code', 'energy_source_code', name=op.f('pk_generation_fuel_by_generator_energy_source_yearly_eia923')) - ) - op.create_table('generation_fuel_by_generator_yearly_eia923', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('fuel_consumed_for_electricity_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel to produce electricity, in physical unit, year to date.'), - sa.Column('fuel_consumed_mmbtu', sa.Float(), nullable=True, comment='Total consumption of fuel in physical unit, year to date. Note: this is the total quantity consumed for both electricity and, in the case of combined heat and power plants, process steam production.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_generation_fuel_by_generator_yearly_eia923_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_generation_fuel_by_generator_yearly_eia923_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_generation_fuel_by_generator_yearly_eia923_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_generation_fuel_by_generator_yearly_eia923')) - ) - op.create_table('heat_rate_by_generator_yearly', - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('fuel_type_count', sa.Integer(), nullable=True, comment='A count of how many different simple energy sources there are associated with a generator.'), - sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_heat_rate_by_generator_yearly_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_heat_rate_by_generator_yearly_prime_mover_code_prime_movers_eia')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_heat_rate_by_generator_yearly')) - ) - op.create_table('mcoe_generators_yearly', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), - sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), - sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), - sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), - sa.Column('utility_name_eia', sa.Text(), nullable=True, comment='The name of the utility.'), - sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), - sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), - sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('generator_operating_date', sa.Date(), nullable=True, comment='Date the generator began commercial operation.'), - sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), - sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), - sa.Column('capacity_mw', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), - sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), - sa.Column('planned_generator_retirement_date', sa.Date(), nullable=True, comment='Planned effective date of the scheduled retirement of the generator.'), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), - sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), - sa.ForeignKeyConstraint(['energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_mcoe_generators_yearly_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_mcoe_generators_yearly_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_mcoe_generators_yearly_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_mcoe_generators_yearly_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_mcoe_generators_yearly_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_mcoe_generators_yearly_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_mcoe_generators_yearly')) - ) - op.create_table('mcoe_yearly', - sa.Column('plant_id_eia', sa.Integer(), nullable=False, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), - sa.Column('generator_id', sa.Text(), nullable=False, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), - sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), - sa.Column('report_date', sa.Date(), nullable=False, comment='Date reported.'), - sa.Column('capacity_factor', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), - sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), - sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), - sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), - sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), - sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), - sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_mcoe_yearly_plant_id_eia_generators_eia860')), - sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_mcoe_yearly')) - ) - op.create_table('mega_generators_eia', + sa.ForeignKeyConstraint(['data_maturity'], ['core_pudl__codes_data_maturities.code'], name=op.f('fk_out_eia__yearly_generators_data_maturity_core_pudl__codes_data_maturities')), + sa.ForeignKeyConstraint(['energy_source_1_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_1_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_1_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_1_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_1_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_1'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_2_transport_1_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_2'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_2_transport_2_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_2_transport_3'], ['core_eia__codes_fuel_transportation_modes.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_2_transport_3_core_eia__codes_fuel_transportation_modes')), + sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_5'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_code_5_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['energy_source_code_6'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_energy_source_code_6_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['operational_status_code'], ['core_eia__codes_operational_status.code'], name=op.f('fk_out_eia__yearly_generators_operational_status_code_core_eia__codes_operational_status')), + sa.ForeignKeyConstraint(['planned_energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_planned_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['planned_new_prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia__yearly_generators_planned_new_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia__yearly_generators_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia__yearly_generators_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia__yearly_generators_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['startup_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_startup_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_2'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_startup_source_code_2_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_3'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_startup_source_code_3_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['startup_source_code_4'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_startup_source_code_4_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia__yearly_generators_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia__yearly_generators_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('report_date', 'plant_id_eia', 'generator_id', name=op.f('pk_out_eia__yearly_generators')) + ) + op.create_table('out_eia__yearly_generators_by_ownership', sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), @@ -3535,7 +3773,7 @@ def upgrade() -> None: sa.Column('fuel_cost_from_eiaapi', sa.Boolean(), nullable=True, comment='Indicates whether the fuel cost was derived from the EIA API.'), sa.Column('fuel_cost_per_mmbtu', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), sa.Column('total_fuel_cost', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), sa.Column('total_mmbtu', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), @@ -3545,27 +3783,27 @@ def upgrade() -> None: sa.Column('capacity_eoy_mw', sa.Float(), nullable=True, comment='Total end of year installed (nameplate) capacity for a plant part, in megawatts.'), sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), sa.Column('ownership_record_type', sa.Enum('owned', 'total'), nullable=True, comment='Whether each generator record is for one owner or represents a total of all ownerships.'), - sa.ForeignKeyConstraint(['energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_mega_generators_eia_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_mega_generators_eia_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_mega_generators_eia_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_mega_generators_eia_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_mega_generators_eia_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_mega_generators_eia_utility_id_pudl_utilities_pudl')) - ) - op.create_table('plant_parts_eia', + sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_generators_by_ownership_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia__yearly_generators_by_ownership_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia__yearly_generators_by_ownership_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia__yearly_generators_by_ownership_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia__yearly_generators_by_ownership_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia__yearly_generators_by_ownership_utility_id_pudl_core_pudl__entity_utilities_pudl')) + ) + op.create_table('out_eia__yearly_plant_parts', sa.Column('record_id_eia', sa.Text(), nullable=False, comment='Identifier for EIA plant parts analysis records.'), sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), - sa.Column('plant_part', sa.Enum('plant_unit', 'plant_prime_mover', 'plant_operating_year', 'plant_ferc_acct', 'plant_match_ferc1', 'plant', 'plant_prime_fuel', 'plant_gen', 'plant_technology'), nullable=True, comment='The part of the plant a record corresponds to.'), + sa.Column('plant_part', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='The part of the plant a record corresponds to.'), sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), - sa.Column('energy_source_code_1', sa.Enum('LIG', 'OTH', 'OG', 'LFG', 'SC', 'MWH', 'WDL', 'MSB', 'SLW', 'WH', 'TDF', 'NUC', 'OBS', 'GEO', 'NG', 'RFO', 'OBL', 'SUB', 'WDS', 'PG', 'SG', 'SGP', 'OBG', 'KER', 'WND', 'DFO', 'WO', 'JF', 'AB', 'MSW', 'ANT', 'RC', 'MSN', 'SGC', 'BFG', 'PUR', 'BLQ', 'BIT', 'PC', 'WAT', 'WC', 'SUN'), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), - sa.Column('technology_description', sa.Enum('Municipal Solid Waste', 'Hydroelectric Pumped Storage', 'Offshore Wind Turbine', 'Petroleum Coke', 'Wood/Wood Waste Biomass', 'Onshore Wind Turbine', 'All Other', 'Hydrokinetic', 'Petroleum Liquids', 'Natural Gas Fired Combustion Turbine', 'Batteries', 'Other Waste Biomass', 'Natural Gas Internal Combustion Engine', 'Landfill Gas', 'Conventional Steam Coal', 'Flywheels', 'Nuclear', 'Coal Integrated Gasification Combined Cycle', 'Conventional Hydroelectric', 'Natural Gas Fired Combined Cycle', 'Other Natural Gas', 'Solar Thermal without Energy Storage', 'Natural Gas with Compressed Air Storage', 'Other Gases', 'Solar Thermal with Energy Storage', 'Solar Photovoltaic', 'Geothermal', 'Natural Gas Steam Turbine'), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), + sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), + sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), sa.Column('ferc_acct_name', sa.Enum('Hydraulic', 'Nuclear', 'Steam', 'Other'), nullable=True, comment='Name of FERC account, derived from technology description and prime mover code.'), sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), sa.Column('true_gran', sa.Boolean(), nullable=True, comment='Indicates whether a plant part list record is associated with the highest priority plant part for all identical records.'), - sa.Column('appro_part_label', sa.Enum('plant_unit', 'plant_prime_mover', 'plant_operating_year', 'plant_ferc_acct', 'plant_match_ferc1', 'plant', 'plant_prime_fuel', 'plant_gen', 'plant_technology'), nullable=True, comment='Plant part of the associated true granularity record.'), + sa.Column('appro_part_label', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='Plant part of the associated true granularity record.'), sa.Column('appro_record_id_eia', sa.Text(), nullable=True, comment='EIA record ID of the associated true granularity record.'), sa.Column('ferc1_generator_agg_id', sa.Integer(), nullable=True, comment='ID dynamically assigned by PUDL to EIA records with multiple matches to a single FERC ID in the FERC-EIA manual matching process.'), sa.Column('capacity_eoy_mw', sa.Float(), nullable=True, comment='Total end of year installed (nameplate) capacity for a plant part, in megawatts.'), @@ -3577,7 +3815,7 @@ def upgrade() -> None: sa.Column('fuel_cost_per_mwh', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), sa.Column('fuel_type_code_pudl', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), sa.Column('generator_retirement_date', sa.Date(), nullable=True, comment='Date of the scheduled or effected retirement of the generator.'), - sa.Column('heat_rate_mmbtu_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), sa.Column('installation_year', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), sa.Column('net_generation_mwh', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), sa.Column('generator_operating_year', sa.Integer(), nullable=True, comment='Year a generator went into service.'), @@ -3596,206 +3834,335 @@ def upgrade() -> None: sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), sa.Column('plant_id_report_year', sa.Text(), nullable=True, comment='PUDL plant ID and report year of the record.'), - sa.ForeignKeyConstraint(['energy_source_code_1'], ['energy_sources_eia.code'], name=op.f('fk_plant_parts_eia_energy_source_code_1_energy_sources_eia')), - sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['generators_eia860.plant_id_eia', 'generators_eia860.generator_id', 'generators_eia860.report_date'], name=op.f('fk_plant_parts_eia_plant_id_eia_generators_eia860')), - sa.ForeignKeyConstraint(['plant_id_pudl'], ['plants_pudl.plant_id_pudl'], name=op.f('fk_plant_parts_eia_plant_id_pudl_plants_pudl')), - sa.ForeignKeyConstraint(['prime_mover_code'], ['prime_movers_eia.code'], name=op.f('fk_plant_parts_eia_prime_mover_code_prime_movers_eia')), - sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['utilities_eia860.utility_id_eia', 'utilities_eia860.report_date'], name=op.f('fk_plant_parts_eia_utility_id_eia_utilities_eia860')), - sa.ForeignKeyConstraint(['utility_id_pudl'], ['utilities_pudl.utility_id_pudl'], name=op.f('fk_plant_parts_eia_utility_id_pudl_utilities_pudl')), - sa.PrimaryKeyConstraint('record_id_eia', name=op.f('pk_plant_parts_eia')) + sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_eia__yearly_plant_parts_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_eia__yearly_plant_parts_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_eia__yearly_plant_parts_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_eia__yearly_plant_parts_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_eia__yearly_plant_parts_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_eia__yearly_plant_parts_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id_eia', name=op.f('pk_out_eia__yearly_plant_parts')) + ) + op.create_table('out_pudl__yearly_assn_eia_ferc1_plant_parts', + sa.Column('record_id_ferc1', sa.Text(), nullable=False, comment='Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.'), + sa.Column('record_id_eia', sa.Text(), nullable=True, comment='Identifier for EIA plant parts analysis records.'), + sa.Column('match_type', sa.Text(), nullable=True, comment='Indicates the source and validation of the match between EIA and FERC. Match types include matches was generated from the model, verified by the training data, overridden by the training data, etc.'), + sa.Column('plant_name_ppe', sa.Text(), nullable=True, comment='Derived plant name that includes EIA plant name and other strings associated with ID and PK columns of the plant part.'), + sa.Column('plant_part', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='The part of the plant a record corresponds to.'), + sa.Column('report_year', sa.Integer(), nullable=True, comment='Four-digit year in which the data was reported.'), + sa.Column('report_date', sa.Date(), nullable=True, comment='Date reported.'), + sa.Column('ownership_record_type', sa.Enum('owned', 'total'), nullable=True, comment='Whether each generator record is for one owner or represents a total of all ownerships.'), + sa.Column('plant_name_eia', sa.Text(), nullable=True, comment='Plant name.'), + sa.Column('plant_id_eia', sa.Integer(), nullable=True, comment='The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.'), + sa.Column('generator_id', sa.Text(), nullable=True, comment='Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!'), + sa.Column('unit_id_pudl', sa.Integer(), nullable=True, comment='Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.'), + sa.Column('prime_mover_code', sa.Text(), nullable=True, comment='Code for the type of prime mover (e.g. CT, CG)'), + sa.Column('energy_source_code_1', sa.Text(), nullable=True, comment='The code representing the most predominant type of energy that fuels the generator.'), + sa.Column('technology_description', sa.Text(), nullable=True, comment='High level description of the technology used by the generator to produce electricity.'), + sa.Column('ferc_acct_name', sa.Enum('Hydraulic', 'Nuclear', 'Steam', 'Other'), nullable=True, comment='Name of FERC account, derived from technology description and prime mover code.'), + sa.Column('generator_operating_year', sa.Integer(), nullable=True, comment='Year a generator went into service.'), + sa.Column('utility_id_eia', sa.Integer(), nullable=True, comment='The EIA Utility Identification number.'), + sa.Column('utility_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL utility ID. May not be stable over time.'), + sa.Column('true_gran', sa.Boolean(), nullable=True, comment='Indicates whether a plant part list record is associated with the highest priority plant part for all identical records.'), + sa.Column('appro_part_label', sa.Enum('plant_unit', 'plant_gen', 'plant', 'plant_operating_year', 'plant_prime_fuel', 'plant_prime_mover', 'plant_technology', 'plant_ferc_acct', 'plant_match_ferc1'), nullable=True, comment='Plant part of the associated true granularity record.'), + sa.Column('appro_record_id_eia', sa.Text(), nullable=True, comment='EIA record ID of the associated true granularity record.'), + sa.Column('record_count', sa.Integer(), nullable=True, comment='Number of distinct generator IDs that partcipated in the aggregation for a plant part list record.'), + sa.Column('fraction_owned', sa.Float(), nullable=True, comment='Proportion of generator ownership attributable to this utility.'), + sa.Column('ownership_dupe', sa.Boolean(), nullable=True, comment='Whether a plant part record has a duplicate record with different ownership status.'), + sa.Column('operational_status', sa.Text(), nullable=True, comment='The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.'), + sa.Column('operational_status_pudl', sa.Enum('operating', 'retired', 'proposed'), nullable=True, comment='The operating status of the asset using PUDL categories.'), + sa.Column('plant_id_pudl', sa.Integer(), nullable=True, comment='A manually assigned PUDL plant ID. May not be constant over time.'), + sa.Column('total_fuel_cost_eia', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('fuel_cost_per_mmbtu_eia', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('net_generation_mwh_eia', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('capacity_mw_eia', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('capacity_factor_eia', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('total_mmbtu_eia', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), + sa.Column('unit_heat_rate_mmbtu_per_mwh_eia', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Coming from MCOE calculation.'), + sa.Column('fuel_type_code_pudl_eia', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('installation_year_eia', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('plant_part_id_eia', sa.Text(), nullable=True, comment='Contains EIA plant ID, plant part, ownership, and EIA utility id'), + sa.Column('utility_id_ferc1', sa.Integer(), nullable=True, comment='PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.'), + sa.Column('utility_name_ferc1', sa.Text(), nullable=True, comment='Name of the responding utility, as it is reported in FERC Form 1. For human readability only.'), + sa.Column('plant_id_ferc1', sa.Integer(), nullable=True, comment='Algorithmically assigned PUDL FERC Plant ID. WARNING: NOT STABLE BETWEEN PUDL DB INITIALIZATIONS.'), + sa.Column('plant_name_ferc1', sa.Text(), nullable=True, comment='Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.'), + sa.Column('asset_retirement_cost', sa.Float(), nullable=True, comment='Asset retirement cost (USD).'), + sa.Column('avg_num_employees', sa.Float(), nullable=True), + sa.Column('capacity_factor_ferc1', sa.Float(), nullable=True, comment='Fraction of potential generation that was actually reported for a plant part.'), + sa.Column('capacity_mw_ferc1', sa.Float(), nullable=True, comment='Total installed (nameplate) capacity, in megawatts.'), + sa.Column('capex_annual_addition', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total`.'), + sa.Column('capex_annual_addition_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total`.'), + sa.Column('capex_annual_per_kw', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per kw.'), + sa.Column('capex_annual_per_mw', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per MW.'), + sa.Column('capex_annual_per_mw_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total` per MW.'), + sa.Column('capex_annual_per_mwh', sa.Float(), nullable=True, comment='Annual capital addition into `capex_total` per MWh.'), + sa.Column('capex_annual_per_mwh_rolling', sa.Float(), nullable=True, comment='Year-to-date capital addition into `capex_total` per MWh.'), + sa.Column('capex_equipment', sa.Float(), nullable=True, comment='Cost of plant: equipment (USD).'), + sa.Column('capex_land', sa.Float(), nullable=True, comment='Cost of plant: land and land rights (USD).'), + sa.Column('capex_per_mw', sa.Float(), nullable=True, comment='Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.'), + sa.Column('capex_structures', sa.Float(), nullable=True, comment='Cost of plant: structures and improvements (USD).'), + sa.Column('capex_total', sa.Float(), nullable=True, comment='Total cost of plant (USD).'), + sa.Column('capex_wo_retirement_total', sa.Float(), nullable=True, comment='Total cost of plant (USD) without retirements.'), + sa.Column('construction_type', sa.Enum('conventional', 'outdoor', 'semioutdoor'), nullable=True, comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings."), + sa.Column('construction_year_eia', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('construction_year_ferc1', sa.Integer(), nullable=True, comment="Year the plant's oldest still operational unit was built."), + sa.Column('installation_year_ferc1', sa.Integer(), nullable=True, comment="Year the plant's most recently built unit was installed."), + sa.Column('net_generation_mwh_ferc1', sa.Float(), nullable=True, comment='Net electricity generation for the specified period in megawatt-hours (MWh).'), + sa.Column('not_water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when not limited by condenser water.'), + sa.Column('opex_allowances', sa.Float(), nullable=True, comment='Allowances.'), + sa.Column('opex_boiler', sa.Float(), nullable=True, comment='Maintenance of boiler (or reactor) plant.'), + sa.Column('opex_coolants', sa.Float(), nullable=True, comment='Cost of coolants and water (nuclear plants only)'), + sa.Column('opex_electric', sa.Float(), nullable=True, comment='Production expenses: electric expenses (USD).'), + sa.Column('opex_engineering', sa.Float(), nullable=True, comment='Production expenses: maintenance, supervision, and engineering (USD).'), + sa.Column('opex_fuel', sa.Float(), nullable=True, comment='Production expenses: fuel (USD).'), + sa.Column('fuel_cost_per_mwh_eia', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.Column('fuel_cost_per_mwh_ferc1', sa.Float(), nullable=True, comment='Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.'), + sa.Column('opex_misc_power', sa.Float(), nullable=True, comment='Miscellaneous steam (or nuclear) expenses.'), + sa.Column('opex_misc_steam', sa.Float(), nullable=True, comment='Maintenance of miscellaneous steam (or nuclear) plant.'), + sa.Column('opex_nonfuel_per_mwh', sa.Float(), nullable=True, comment='Investments in non-fuel production expenses per Mwh.'), + sa.Column('opex_operations', sa.Float(), nullable=True, comment='Production expenses: operations, supervision, and engineering (USD).'), + sa.Column('opex_per_mwh', sa.Float(), nullable=True, comment='Total production expenses (USD per MWh generated).'), + sa.Column('opex_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of electric plant (USD).'), + sa.Column('opex_production_total', sa.Float(), nullable=True, comment='Total operating expenses.'), + sa.Column('opex_rents', sa.Float(), nullable=True, comment='Production expenses: rents (USD).'), + sa.Column('opex_steam', sa.Float(), nullable=True, comment='Steam expenses.'), + sa.Column('opex_steam_other', sa.Float(), nullable=True, comment='Steam from other sources.'), + sa.Column('opex_structures', sa.Float(), nullable=True, comment='Production expenses: maintenance of structures (USD).'), + sa.Column('opex_total_nonfuel', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('opex_transfer', sa.Float(), nullable=True, comment='Steam transferred (Credit).'), + sa.Column('peak_demand_mw', sa.Float(), nullable=True, comment='Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.'), + sa.Column('plant_capability_mw', sa.Float(), nullable=True, comment='Net plant capability in megawatts.'), + sa.Column('plant_hours_connected_while_generating', sa.Float(), nullable=True, comment='Hours the plant was connected to load while generating in the report year.'), + sa.Column('plant_type', sa.Text(), nullable=True), + sa.Column('water_limited_capacity_mw', sa.Float(), nullable=True, comment='Plant capacity in MW when limited by condenser water.'), + sa.Column('fuel_cost_per_mmbtu_ferc1', sa.Float(), nullable=True, comment='Average fuel cost per mmBTU of heat content in nominal USD.'), + sa.Column('fuel_type', sa.Text(), nullable=True), + sa.Column('license_id_ferc1', sa.Integer(), nullable=True, comment='FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.'), + sa.Column('opex_maintenance', sa.Float(), nullable=True, comment='Production expenses: Maintenance (USD).'), + sa.Column('opex_total', sa.Float(), nullable=True, comment='Total production expenses, excluding fuel (USD).'), + sa.Column('capex_facilities', sa.Float(), nullable=True, comment='Cost of plant: reservoirs, dams, and waterways (USD).'), + sa.Column('capex_roads', sa.Float(), nullable=True, comment='Cost of plant: roads, railroads, and bridges (USD).'), + sa.Column('net_capacity_adverse_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the least favorable operating conditions, in megawatts.'), + sa.Column('net_capacity_favorable_conditions_mw', sa.Float(), nullable=True, comment='Net plant capability under the most favorable operating conditions, in megawatts.'), + sa.Column('opex_dams', sa.Float(), nullable=True, comment='Production expenses: maintenance of reservoirs, dams, and waterways (USD).'), + sa.Column('opex_generation_misc', sa.Float(), nullable=True, comment='Production expenses: miscellaneous power generation expenses (USD).'), + sa.Column('opex_hydraulic', sa.Float(), nullable=True, comment='Production expenses: hydraulic expenses (USD).'), + sa.Column('opex_misc_plant', sa.Float(), nullable=True, comment='Production expenses: maintenance of miscellaneous hydraulic plant (USD).'), + sa.Column('opex_water_for_power', sa.Float(), nullable=True, comment='Production expenses: water for power (USD).'), + sa.Column('ferc_license_id', sa.Text(), nullable=True, comment='The FERC license ID of a project.'), + sa.Column('capex_equipment_electric', sa.Float(), nullable=True, comment='Cost of plant: accessory electric equipment (USD).'), + sa.Column('capex_equipment_misc', sa.Float(), nullable=True, comment='Cost of plant: miscellaneous power plant equipment (USD).'), + sa.Column('capex_wheels_turbines_generators', sa.Float(), nullable=True, comment='Cost of plant: water wheels, turbines, and generators (USD).'), + sa.Column('energy_used_for_pumping_mwh', sa.Float(), nullable=True, comment='Energy used for pumping, in megawatt-hours.'), + sa.Column('net_load_mwh', sa.Float(), nullable=True, comment='Net output for load (net generation - energy used for pumping) in megawatt-hours.'), + sa.Column('opex_production_before_pumping', sa.Float(), nullable=True, comment='Total production expenses before pumping (USD).'), + sa.Column('opex_pumped_storage', sa.Float(), nullable=True, comment='Production expenses: pumped storage (USD).'), + sa.Column('opex_pumping', sa.Float(), nullable=True, comment='Production expenses: We are here to PUMP YOU UP! (USD).'), + sa.Column('total_fuel_cost_ferc1', sa.Float(), nullable=True, comment='Total annual reported fuel costs for the plant part. Includes costs from all fuels.'), + sa.Column('total_mmbtu_ferc1', sa.Float(), nullable=True, comment='Total annual heat content of fuel consumed by a plant part record in the plant parts list.'), + sa.Column('fuel_type_code_pudl_ferc1', sa.Enum('coal', 'gas', 'hydro', 'nuclear', 'oil', 'other', 'solar', 'waste', 'wind'), nullable=True, comment='Simplified fuel type code used in PUDL'), + sa.Column('unit_heat_rate_mmbtu_per_mwh_ferc1', sa.Float(), nullable=True, comment='Fuel content per unit of electricity generated. Calculated from FERC reported fuel consumption and net generation.'), + sa.ForeignKeyConstraint(['energy_source_code_1'], ['core_eia__codes_energy_sources.code'], name=op.f('fk_out_pudl__yearly_assn_eia_ferc1_plant_parts_energy_source_code_1_core_eia__codes_energy_sources')), + sa.ForeignKeyConstraint(['plant_id_eia', 'generator_id', 'report_date'], ['core_eia860__scd_generators.plant_id_eia', 'core_eia860__scd_generators.generator_id', 'core_eia860__scd_generators.report_date'], name=op.f('fk_out_pudl__yearly_assn_eia_ferc1_plant_parts_plant_id_eia_core_eia860__scd_generators')), + sa.ForeignKeyConstraint(['plant_id_pudl'], ['core_pudl__entity_plants_pudl.plant_id_pudl'], name=op.f('fk_out_pudl__yearly_assn_eia_ferc1_plant_parts_plant_id_pudl_core_pudl__entity_plants_pudl')), + sa.ForeignKeyConstraint(['prime_mover_code'], ['core_eia__codes_prime_movers.code'], name=op.f('fk_out_pudl__yearly_assn_eia_ferc1_plant_parts_prime_mover_code_core_eia__codes_prime_movers')), + sa.ForeignKeyConstraint(['utility_id_eia', 'report_date'], ['core_eia860__scd_utilities.utility_id_eia', 'core_eia860__scd_utilities.report_date'], name=op.f('fk_out_pudl__yearly_assn_eia_ferc1_plant_parts_utility_id_eia_core_eia860__scd_utilities')), + sa.ForeignKeyConstraint(['utility_id_ferc1', 'plant_name_ferc1'], ['core_pudl__assn_ferc1_pudl_plants.utility_id_ferc1', 'core_pudl__assn_ferc1_pudl_plants.plant_name_ferc1'], name=op.f('fk_out_pudl__yearly_assn_eia_ferc1_plant_parts_utility_id_ferc1_core_pudl__assn_ferc1_pudl_plants')), + sa.ForeignKeyConstraint(['utility_id_pudl'], ['core_pudl__entity_utilities_pudl.utility_id_pudl'], name=op.f('fk_out_pudl__yearly_assn_eia_ferc1_plant_parts_utility_id_pudl_core_pudl__entity_utilities_pudl')), + sa.PrimaryKeyConstraint('record_id_ferc1', name=op.f('pk_out_pudl__yearly_assn_eia_ferc1_plant_parts')) ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('plant_parts_eia') - op.drop_table('mega_generators_eia') - op.drop_table('mcoe_yearly') - op.drop_table('mcoe_generators_yearly') - op.drop_table('heat_rate_by_generator_yearly') - op.drop_table('generation_fuel_by_generator_yearly_eia923') - op.drop_table('generation_fuel_by_generator_energy_source_yearly_eia923') - op.drop_table('generation_fuel_by_generator_energy_source_owner_yearly_eia923') - op.drop_table('fuel_cost_by_generator_yearly') - op.drop_table('denorm_ownership_eia860') - op.drop_table('denorm_generators_eia') - op.drop_table('denorm_generation_yearly_eia923') - op.drop_table('denorm_boilers_eia') - op.drop_table('denorm_boiler_fuel_yearly_eia923') - op.drop_table('capacity_factor_by_generator_yearly') - op.drop_table('boiler_stack_flue_assn_eia860') - op.drop_table('boiler_generator_assn_eia860') - op.drop_table('boiler_emissions_control_equipment_assn_eia860') - op.drop_table('boiler_cooling_assn_eia860') - op.drop_table('plants_steam_ferc1') - op.drop_table('plants_small_ferc1') - op.drop_table('plants_pumped_storage_ferc1') - op.drop_table('plants_hydro_ferc1') - op.drop_table('heat_rate_by_unit_yearly') - op.drop_table('generators_eia860') - op.drop_table('fuel_ferc1') - op.drop_table('denorm_plants_utilities_ferc1') - op.drop_table('denorm_plants_utilities_eia') - op.drop_table('denorm_plants_steam_ferc1') - op.drop_table('denorm_plants_small_ferc1') - op.drop_table('denorm_plants_pumped_storage_ferc1') - op.drop_table('denorm_plants_hydro_ferc1') - op.drop_table('denorm_plants_eia') - op.drop_table('denorm_plants_all_ferc1') - op.drop_table('denorm_generation_fuel_combined_yearly_eia923') - op.drop_table('denorm_fuel_receipts_costs_yearly_eia923') - op.drop_table('denorm_fuel_ferc1') - op.drop_table('denorm_fuel_by_plant_ferc1') - op.drop_table('boilers_eia860') - op.drop_table('utility_plant_summary_ferc1') - op.drop_table('utilities_ferc1_xbrl') - op.drop_table('utilities_ferc1_dbf') - op.drop_table('transmission_statistics_ferc1') - op.drop_table('retained_earnings_ferc1') - op.drop_table('purchased_power_ferc1') - op.drop_table('plants_ferc1') - op.drop_table('plants_eia860') - op.drop_table('plant_in_service_ferc1') - op.drop_table('ownership_eia860') - op.drop_table('other_regulatory_liabilities_ferc1') - op.drop_table('mcoe_monthly') - op.drop_table('mcoe_generators_monthly') - op.drop_table('income_statement_ferc1') - op.drop_table('heat_rate_by_generator_monthly') - op.drop_table('generation_fuel_by_generator_monthly_eia923') - op.drop_table('generation_fuel_by_generator_energy_source_monthly_eia923') - op.drop_table('generation_eia923') - op.drop_table('fuel_receipts_costs_eia923') - op.drop_table('fuel_cost_by_generator_monthly') - op.drop_table('epacamd_eia') - op.drop_table('electricity_sales_by_rate_schedule_ferc1') - op.drop_table('electric_plant_depreciation_functional_ferc1') - op.drop_table('electric_plant_depreciation_changes_ferc1') - op.drop_table('electric_operating_revenues_ferc1') - op.drop_table('electric_operating_expenses_ferc1') - op.drop_table('electric_energy_sources_ferc1') - op.drop_table('electric_energy_dispositions_ferc1') - op.drop_table('depreciation_amortization_summary_ferc1') - op.drop_table('denorm_utility_plant_summary_ferc1') - op.drop_table('denorm_utilities_eia') - op.drop_table('denorm_transmission_statistics_ferc1') - op.drop_table('denorm_retained_earnings_ferc1') - op.drop_table('denorm_purchased_power_ferc1') - op.drop_table('denorm_plant_in_service_ferc1') - op.drop_table('denorm_other_regulatory_liabilities_ferc1') - op.drop_table('denorm_income_statement_ferc1') - op.drop_table('denorm_generation_monthly_eia923') - op.drop_table('denorm_generation_eia923') - op.drop_table('denorm_electricity_sales_by_rate_schedule_ferc1') - op.drop_table('denorm_electric_plant_depreciation_functional_ferc1') - op.drop_table('denorm_electric_plant_depreciation_changes_ferc1') - op.drop_table('denorm_electric_operating_revenues_ferc1') - op.drop_table('denorm_electric_operating_expenses_ferc1') - op.drop_table('denorm_electric_energy_sources_ferc1') - op.drop_table('denorm_electric_energy_dispositions_ferc1') - op.drop_table('denorm_depreciation_amortization_summary_ferc1') - op.drop_table('denorm_cash_flow_ferc1') - op.drop_table('denorm_boiler_fuel_monthly_eia923') - op.drop_table('denorm_boiler_fuel_eia923') - op.drop_table('denorm_balance_sheet_liabilities_ferc1') - op.drop_table('denorm_balance_sheet_assets_ferc1') - op.drop_table('cash_flow_ferc1') - op.drop_table('capacity_factor_by_generator_monthly') - op.drop_table('boiler_fuel_eia923') - op.drop_table('balance_sheet_liabilities_ferc1') - op.drop_table('balance_sheet_assets_ferc1') - op.drop_table('utility_plant_assn') - op.drop_table('utility_data_rto_eia861') - op.drop_table('utility_data_nerc_eia861') - op.drop_table('utility_data_misc_eia861') - op.drop_table('utilities_ferc1') - op.drop_table('utilities_eia860') - op.drop_table('utilities_eia') - op.drop_table('summarized_demand_ferc714') - op.drop_table('service_territory_eia861') - op.drop_table('sales_eia861') - op.drop_table('reliability_eia861') - op.drop_table('plants_eia') - op.drop_table('operational_data_revenue_eia861') - op.drop_table('operational_data_misc_eia861') - op.drop_table('non_net_metering_misc_eia861') - op.drop_table('non_net_metering_customer_fuel_class_eia861') - op.drop_table('net_metering_misc_eia861') - op.drop_table('net_metering_customer_fuel_class_eia861') - op.drop_table('mergers_eia861') - op.drop_table('heat_rate_by_unit_monthly') - op.drop_table('green_pricing_eia861') - op.drop_table('generators_entity_eia') - op.drop_table('generation_fuel_nuclear_eia923') - op.drop_table('generation_fuel_eia923') - op.drop_table('fipsified_respondents_ferc714') - op.drop_table('energy_efficiency_eia861') - op.drop_table('emissions_control_equipment_eia860') - op.drop_table('dynamic_pricing_eia861') - op.drop_table('distribution_systems_eia861') - op.drop_table('distributed_generation_tech_eia861') - op.drop_table('distributed_generation_misc_eia861') - op.drop_table('distributed_generation_fuel_eia861') - op.drop_table('denorm_generation_fuel_combined_monthly_eia923') - op.drop_table('denorm_generation_fuel_combined_eia923') - op.drop_table('denorm_fuel_receipts_costs_monthly_eia923') - op.drop_table('denorm_fuel_receipts_costs_eia923') - op.drop_table('denorm_emissions_control_equipment_eia860') - op.drop_table('demand_side_management_sales_eia861') - op.drop_table('demand_side_management_misc_eia861') - op.drop_table('demand_side_management_ee_dr_eia861') - op.drop_table('demand_response_water_heater_eia861') - op.drop_table('demand_response_eia861') - op.drop_table('demand_hourly_pa_ferc714') - op.drop_table('coalmine_eia923') - op.drop_table('boilers_entity_eia') - op.drop_table('advanced_metering_infrastructure_eia861') - op.drop_table('wet_dry_bottom_eia') - op.drop_table('utility_assn_eia861') - op.drop_table('utilities_pudl') - op.drop_table('utilities_entity_eia') - op.drop_table('steam_plant_types_eia') - op.drop_table('so2_units_eia') - op.drop_table('so2_compliance_strategies_eia') - op.drop_table('sector_consolidated_eia') - op.drop_table('respondent_id_ferc714') - op.drop_table('reporting_frequencies_eia') - op.drop_table('regulations_eia') - op.drop_table('prime_movers_eia') - op.drop_table('predicted_state_hourly_demand') - op.drop_table('power_purchase_types_ferc1') - op.drop_table('political_subdivisions') - op.drop_table('plants_pudl') - op.drop_table('plants_entity_eia') - op.drop_table('particulate_units_eia') - op.drop_table('particulate_compliance_strategies_eia') - op.drop_table('operational_status_eia') - op.drop_table('nox_units_eia') - op.drop_table('nox_control_status_eia') - op.drop_table('nox_compliance_strategies_eia') - op.drop_table('momentary_interruptions_eia') - op.drop_table('mercury_compliance_strategies_eia') - op.drop_table('fuel_types_aer_eia') - op.drop_table('fuel_transportation_modes_eia') - op.drop_table('fuel_receipts_costs_aggs_eia') - op.drop_table('firing_types_eia') - op.drop_table('ferc_accounts') - op.drop_table('epacamd_eia_subplant_ids') - op.drop_table('environmental_equipment_manufacturers_eia') - op.drop_table('energy_sources_eia') - op.drop_table('emission_control_equipment_types_eia') - op.drop_table('datasources') - op.drop_table('data_maturities') - op.drop_table('contract_types_eia') - op.drop_table('compiled_geometry_utility_eia861') - op.drop_table('compiled_geometry_balancing_authority_eia861') - op.drop_table('coalmine_types_eia') - op.drop_table('boiler_types_eia') - op.drop_table('boiler_status_eia') - op.drop_table('boiler_generator_assn_types_eia') - op.drop_table('balancing_authority_eia861') - op.drop_table('balancing_authority_assn_eia861') - op.drop_table('balancing_authorities_eia') - op.drop_table('averaging_periods_eia') + op.drop_table('out_pudl__yearly_assn_eia_ferc1_plant_parts') + op.drop_table('out_eia__yearly_plant_parts') + op.drop_table('out_eia__yearly_generators_by_ownership') + op.drop_table('out_eia__yearly_generators') + op.drop_table('out_eia__yearly_boilers') + op.drop_table('out_eia923__yearly_generation_fuel_by_generator_energy_source_owner') + op.drop_table('out_eia923__yearly_generation_fuel_by_generator_energy_source') + op.drop_table('out_eia923__yearly_generation_fuel_by_generator') + op.drop_table('out_eia923__yearly_generation') + op.drop_table('out_eia923__yearly_boiler_fuel') + op.drop_table('out_eia860__yearly_ownership') + op.drop_table('core_eia860__scd_ownership') + op.drop_table('core_eia860__assn_yearly_boiler_emissions_control_equipment') + op.drop_table('core_eia860__assn_boiler_stack_flue') + op.drop_table('core_eia860__assn_boiler_generator') + op.drop_table('core_eia860__assn_boiler_cooling') + op.drop_table('_out_eia__yearly_heat_rate_by_generator') + op.drop_table('_out_eia__yearly_generators') + op.drop_table('_out_eia__yearly_fuel_cost_by_generator') + op.drop_table('_out_eia__yearly_derived_generator_attributes') + op.drop_table('_out_eia__yearly_capacity_factor_by_generator') + op.drop_table('out_ferc1__yearly_steam_plants_fuel_sched402') + op.drop_table('out_ferc1__yearly_steam_plants_fuel_by_plant_sched402') + op.drop_table('out_ferc1__yearly_all_plants') + op.drop_table('out_eia__yearly_plants') + op.drop_table('out_eia923__yearly_generation_fuel_combined') + op.drop_table('out_eia923__yearly_fuel_receipts_costs') + op.drop_table('core_ferc1__yearly_steam_plants_sched402') + op.drop_table('core_ferc1__yearly_steam_plants_fuel_sched402') + op.drop_table('core_ferc1__yearly_small_plants_sched410') + op.drop_table('core_ferc1__yearly_pumped_storage_plants_sched408') + op.drop_table('core_ferc1__yearly_hydroelectric_plants_sched406') + op.drop_table('core_eia860__scd_generators') + op.drop_table('core_eia860__scd_boilers') + op.drop_table('_out_ferc1__yearly_steam_plants_sched402') + op.drop_table('_out_ferc1__yearly_small_plants_sched410') + op.drop_table('_out_ferc1__yearly_pumped_storage_plants_sched408') + op.drop_table('_out_ferc1__yearly_plants_utilities') + op.drop_table('_out_ferc1__yearly_hydroelectric_plants_sched406') + op.drop_table('_out_eia__yearly_heat_rate_by_unit') + op.drop_table('_out_eia__plants_utilities') + op.drop_table('out_ferc1__yearly_utility_plant_summary_sched200') + op.drop_table('out_ferc1__yearly_transmission_lines_sched422') + op.drop_table('out_ferc1__yearly_sales_by_rate_schedules_sched304') + op.drop_table('out_ferc1__yearly_retained_earnings_sched118') + op.drop_table('out_ferc1__yearly_purchased_power_and_exchanges_sched326') + op.drop_table('out_ferc1__yearly_plant_in_service_sched204') + op.drop_table('out_ferc1__yearly_other_regulatory_liabilities_sched278') + op.drop_table('out_ferc1__yearly_operating_revenues_sched300') + op.drop_table('out_ferc1__yearly_operating_expenses_sched320') + op.drop_table('out_ferc1__yearly_income_statements_sched114') + op.drop_table('out_ferc1__yearly_energy_sources_sched401') + op.drop_table('out_ferc1__yearly_energy_dispositions_sched401') + op.drop_table('out_ferc1__yearly_depreciation_summary_sched336') + op.drop_table('out_ferc1__yearly_depreciation_changes_sched219') + op.drop_table('out_ferc1__yearly_depreciation_by_function_sched219') + op.drop_table('out_ferc1__yearly_cash_flows_sched120') + op.drop_table('out_ferc1__yearly_balance_sheet_liabilities_sched110') + op.drop_table('out_ferc1__yearly_balance_sheet_assets_sched110') + op.drop_table('out_eia__yearly_utilities') + op.drop_table('out_eia__monthly_generators') + op.drop_table('out_eia923__monthly_generation_fuel_by_generator_energy_source') + op.drop_table('out_eia923__monthly_generation_fuel_by_generator') + op.drop_table('out_eia923__monthly_generation') + op.drop_table('out_eia923__monthly_boiler_fuel') + op.drop_table('out_eia923__generation') + op.drop_table('out_eia923__boiler_fuel') + op.drop_table('core_pudl__assn_ferc1_xbrl_pudl_utilities') + op.drop_table('core_pudl__assn_ferc1_pudl_plants') + op.drop_table('core_pudl__assn_ferc1_dbf_pudl_utilities') + op.drop_table('core_ferc1__yearly_utility_plant_summary_sched200') + op.drop_table('core_ferc1__yearly_transmission_lines_sched422') + op.drop_table('core_ferc1__yearly_sales_by_rate_schedules_sched304') + op.drop_table('core_ferc1__yearly_retained_earnings_sched118') + op.drop_table('core_ferc1__yearly_purchased_power_and_exchanges_sched326') + op.drop_table('core_ferc1__yearly_plant_in_service_sched204') + op.drop_table('core_ferc1__yearly_other_regulatory_liabilities_sched278') + op.drop_table('core_ferc1__yearly_operating_revenues_sched300') + op.drop_table('core_ferc1__yearly_operating_expenses_sched320') + op.drop_table('core_ferc1__yearly_income_statements_sched114') + op.drop_table('core_ferc1__yearly_energy_sources_sched401') + op.drop_table('core_ferc1__yearly_energy_dispositions_sched401') + op.drop_table('core_ferc1__yearly_depreciation_summary_sched336') + op.drop_table('core_ferc1__yearly_depreciation_changes_sched219') + op.drop_table('core_ferc1__yearly_depreciation_by_function_sched219') + op.drop_table('core_ferc1__yearly_cash_flows_sched120') + op.drop_table('core_ferc1__yearly_balance_sheet_liabilities_sched110') + op.drop_table('core_ferc1__yearly_balance_sheet_assets_sched110') + op.drop_table('core_epa__assn_eia_epacamd') + op.drop_table('core_eia923__monthly_generation') + op.drop_table('core_eia923__monthly_fuel_receipts_costs') + op.drop_table('core_eia923__monthly_boiler_fuel') + op.drop_table('core_eia860__scd_plants') + op.drop_table('_out_eia__monthly_heat_rate_by_generator') + op.drop_table('_out_eia__monthly_fuel_cost_by_generator') + op.drop_table('_out_eia__monthly_derived_generator_attributes') + op.drop_table('_out_eia__monthly_capacity_factor_by_generator') + op.drop_table('out_ferc714__summarized_demand') + op.drop_table('out_ferc714__respondents_with_fips') + op.drop_table('out_eia923__monthly_generation_fuel_combined') + op.drop_table('out_eia923__monthly_fuel_receipts_costs') + op.drop_table('out_eia923__generation_fuel_combined') + op.drop_table('out_eia923__fuel_receipts_costs') + op.drop_table('out_eia860__yearly_emissions_control_equipment') + op.drop_table('core_pudl__assn_utilities_plants') + op.drop_table('core_pudl__assn_ferc1_pudl_utilities') + op.drop_table('core_pudl__assn_eia_pudl_utilities') + op.drop_table('core_pudl__assn_eia_pudl_plants') + op.drop_table('core_ferc714__hourly_demand_pa') + op.drop_table('core_eia__entity_generators') + op.drop_table('core_eia__entity_boilers') + op.drop_table('core_eia923__monthly_generation_fuel_nuclear') + op.drop_table('core_eia923__monthly_generation_fuel') + op.drop_table('core_eia923__entity_coalmine') + op.drop_table('core_eia861__yearly_utility_data_rto') + op.drop_table('core_eia861__yearly_utility_data_nerc') + op.drop_table('core_eia861__yearly_utility_data_misc') + op.drop_table('core_eia861__yearly_service_territory') + op.drop_table('core_eia861__yearly_sales') + op.drop_table('core_eia861__yearly_reliability') + op.drop_table('core_eia861__yearly_operational_data_revenue') + op.drop_table('core_eia861__yearly_operational_data_misc') + op.drop_table('core_eia861__yearly_non_net_metering_misc') + op.drop_table('core_eia861__yearly_non_net_metering_customer_fuel_class') + op.drop_table('core_eia861__yearly_net_metering_misc') + op.drop_table('core_eia861__yearly_net_metering_customer_fuel_class') + op.drop_table('core_eia861__yearly_mergers') + op.drop_table('core_eia861__yearly_green_pricing') + op.drop_table('core_eia861__yearly_energy_efficiency') + op.drop_table('core_eia861__yearly_dynamic_pricing') + op.drop_table('core_eia861__yearly_distribution_systems') + op.drop_table('core_eia861__yearly_distributed_generation_tech') + op.drop_table('core_eia861__yearly_distributed_generation_misc') + op.drop_table('core_eia861__yearly_distributed_generation_fuel') + op.drop_table('core_eia861__yearly_demand_side_management_sales') + op.drop_table('core_eia861__yearly_demand_side_management_misc') + op.drop_table('core_eia861__yearly_demand_side_management_ee_dr') + op.drop_table('core_eia861__yearly_demand_response_water_heater') + op.drop_table('core_eia861__yearly_demand_response') + op.drop_table('core_eia861__yearly_advanced_metering_infrastructure') + op.drop_table('core_eia860__scd_utilities') + op.drop_table('core_eia860__scd_emissions_control_equipment') + op.drop_table('_out_eia__monthly_heat_rate_by_unit') + op.drop_table('out_ferc714__hourly_predicted_state_demand') + op.drop_table('out_eia861__compiled_geometry_utilities') + op.drop_table('out_eia861__compiled_geometry_balancing_authorities') + op.drop_table('core_pudl__entity_utilities_pudl') + op.drop_table('core_pudl__entity_plants_pudl') + op.drop_table('core_pudl__codes_subdivisions') + op.drop_table('core_pudl__codes_datasources') + op.drop_table('core_pudl__codes_data_maturities') + op.drop_table('core_ferc__codes_accounts') + op.drop_table('core_ferc714__respondent_id') + op.drop_table('core_ferc1__codes_power_purchase_types') + op.drop_table('core_epa__assn_eia_epacamd_subplant_ids') + op.drop_table('core_eia__yearly_fuel_receipts_costs_aggs') + op.drop_table('core_eia__entity_utilities') + op.drop_table('core_eia__entity_plants') + op.drop_table('core_eia__codes_wet_dry_bottom') + op.drop_table('core_eia__codes_steam_plant_types') + op.drop_table('core_eia__codes_so2_units') + op.drop_table('core_eia__codes_so2_compliance_strategies') + op.drop_table('core_eia__codes_sector_consolidated') + op.drop_table('core_eia__codes_reporting_frequencies') + op.drop_table('core_eia__codes_regulations') + op.drop_table('core_eia__codes_prime_movers') + op.drop_table('core_eia__codes_particulate_units') + op.drop_table('core_eia__codes_particulate_compliance_strategies') + op.drop_table('core_eia__codes_operational_status') + op.drop_table('core_eia__codes_nox_units') + op.drop_table('core_eia__codes_nox_control_status') + op.drop_table('core_eia__codes_nox_compliance_strategies') + op.drop_table('core_eia__codes_momentary_interruptions') + op.drop_table('core_eia__codes_mercury_compliance_strategies') + op.drop_table('core_eia__codes_fuel_types_aer') + op.drop_table('core_eia__codes_fuel_transportation_modes') + op.drop_table('core_eia__codes_firing_types') + op.drop_table('core_eia__codes_environmental_equipment_manufacturers') + op.drop_table('core_eia__codes_energy_sources') + op.drop_table('core_eia__codes_emission_control_equipment_types') + op.drop_table('core_eia__codes_contract_types') + op.drop_table('core_eia__codes_coalmine_types') + op.drop_table('core_eia__codes_boiler_types') + op.drop_table('core_eia__codes_boiler_status') + op.drop_table('core_eia__codes_boiler_generator_assn_types') + op.drop_table('core_eia__codes_balancing_authorities') + op.drop_table('core_eia__codes_averaging_periods') + op.drop_table('core_eia861__yearly_balancing_authority') + op.drop_table('core_eia861__assn_utility') + op.drop_table('core_eia861__assn_balancing_authority') # ### end Alembic commands ### diff --git a/migrations/versions/7fa2763bd630_add_owner_utility_fk_rule.py b/migrations/versions/7fa2763bd630_add_owner_utility_fk_rule.py deleted file mode 100644 index d7ef9a6b60..0000000000 --- a/migrations/versions/7fa2763bd630_add_owner_utility_fk_rule.py +++ /dev/null @@ -1,164 +0,0 @@ -"""add owner utility fk rule - -Revision ID: 7fa2763bd630 -Revises: a3dcde9d7419 -Create Date: 2023-10-04 23:22:10.684350 - -""" -from alembic import op - -# revision identifiers, used by Alembic. -revision = '7fa2763bd630' -down_revision = 'a3dcde9d7419' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('denorm_boiler_fuel_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_boilers_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_boilers_eia_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_boilers_eia_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_fuel_receipts_costs_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_generation_fuel_combined_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_generation_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_generators_eia_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_generators_eia_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.create_foreign_key(batch_op.f('fk_denorm_ownership_eia860_owner_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['owner_utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_plants_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_plants_eia_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_plants_eia_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_plants_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_utilities_eia_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_utilities_eia_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('fuel_cost_by_generator_yearly', schema=None) as batch_op: - batch_op.drop_constraint('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('generation_fuel_by_generator_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('generators_eia860', schema=None) as batch_op: - batch_op.drop_constraint('fk_generators_eia860_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_generators_eia860_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('mcoe_generators_yearly', schema=None) as batch_op: - batch_op.drop_constraint('fk_mcoe_generators_yearly_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_mcoe_generators_yearly_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('mega_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_mega_generators_eia_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_mega_generators_eia_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('ownership_eia860', schema=None) as batch_op: - batch_op.create_foreign_key(batch_op.f('fk_ownership_eia860_owner_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['owner_utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('plant_parts_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_plant_parts_eia_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_plant_parts_eia_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('plants_eia860', schema=None) as batch_op: - batch_op.drop_constraint('fk_plants_eia860_utility_id_eia_utilities_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_plants_eia860_utility_id_eia_utilities_eia860'), 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('plants_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_plants_eia860_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_plants_eia860_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('plant_parts_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_plant_parts_eia_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_plant_parts_eia_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_ownership_eia860_owner_utility_id_eia_utilities_eia860'), type_='foreignkey') - - with op.batch_alter_table('mega_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_mega_generators_eia_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_mega_generators_eia_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('mcoe_generators_yearly', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_mcoe_generators_yearly_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_mcoe_generators_yearly_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('generators_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_generators_eia860_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_generators_eia860_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('generation_fuel_by_generator_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('fuel_cost_by_generator_yearly', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_utilities_eia_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_utilities_eia_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_plants_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_plants_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_plants_eia_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_plants_eia_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_ownership_eia860_owner_utility_id_eia_utilities_eia860'), type_='foreignkey') - - with op.batch_alter_table('denorm_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generators_eia_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_generators_eia_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_generation_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_generation_fuel_combined_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_fuel_receipts_costs_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_boilers_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_boilers_eia_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_boilers_eia_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_boiler_fuel_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_entity_eia', 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - # ### end Alembic commands ### diff --git a/migrations/versions/7febe79b8760_add_utility_type_to_liabilities_side.py b/migrations/versions/7febe79b8760_add_utility_type_to_liabilities_side.py deleted file mode 100644 index 91c942f415..0000000000 --- a/migrations/versions/7febe79b8760_add_utility_type_to_liabilities_side.py +++ /dev/null @@ -1,89 +0,0 @@ -"""add utility type to liabilities side - -Revision ID: 7febe79b8760 -Revises: ad3e7ecdcfc3 -Create Date: 2023-09-28 11:10:01.521413 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "7febe79b8760" -down_revision = "ad3e7ecdcfc3" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table( - "balance_sheet_liabilities_ferc1", schema=None - ) as batch_op: - batch_op.add_column( - sa.Column( - "utility_type", - sa.Text(), - nullable=True, - comment="Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.", - ) - ) - - with op.batch_alter_table( - "denorm_balance_sheet_liabilities_ferc1", schema=None - ) as batch_op: - batch_op.add_column( - sa.Column( - "utility_type", - sa.Text(), - nullable=True, - comment="Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.", - ) - ) - - with op.batch_alter_table( - "denorm_retained_earnings_ferc1", schema=None - ) as batch_op: - batch_op.add_column( - sa.Column( - "utility_type", - sa.Text(), - nullable=True, - comment="Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.", - ) - ) - - with op.batch_alter_table("retained_earnings_ferc1", schema=None) as batch_op: - batch_op.add_column( - sa.Column( - "utility_type", - sa.Text(), - nullable=True, - comment="Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.", - ) - ) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table("retained_earnings_ferc1", schema=None) as batch_op: - batch_op.drop_column("utility_type") - - with op.batch_alter_table( - "denorm_retained_earnings_ferc1", schema=None - ) as batch_op: - batch_op.drop_column("utility_type") - - with op.batch_alter_table( - "denorm_balance_sheet_liabilities_ferc1", schema=None - ) as batch_op: - batch_op.drop_column("utility_type") - - with op.batch_alter_table( - "balance_sheet_liabilities_ferc1", schema=None - ) as batch_op: - batch_op.drop_column("utility_type") - - # ### end Alembic commands ### diff --git a/migrations/versions/a3dcde9d7419_update_foreign_key_rules.py b/migrations/versions/a3dcde9d7419_update_foreign_key_rules.py deleted file mode 100644 index 903af9d494..0000000000 --- a/migrations/versions/a3dcde9d7419_update_foreign_key_rules.py +++ /dev/null @@ -1,168 +0,0 @@ -"""update foreign key rules - -Revision ID: a3dcde9d7419 -Revises: 0678b90669bb -Create Date: 2023-10-04 20:52:41.479444 - -""" -from alembic import op - -# revision identifiers, used by Alembic. -revision = 'a3dcde9d7419' -down_revision = '0678b90669bb' -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('denorm_boiler_fuel_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_boilers_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_boilers_eia_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_boilers_eia_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_fuel_receipts_costs_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_generation_fuel_combined_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_generation_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_generators_eia_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_generators_eia_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_ownership_eia860_plant_id_eia_generators_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_ownership_eia860_plant_id_eia_generators_eia860'), 'generators_eia860', ['plant_id_eia', 'generator_id', 'report_date'], ['plant_id_eia', 'generator_id', 'report_date']) - - with op.batch_alter_table('denorm_plants_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_plants_eia_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_plants_eia_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_plants_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('denorm_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_denorm_utilities_eia_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_denorm_utilities_eia_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('fuel_cost_by_generator_yearly', schema=None) as batch_op: - batch_op.drop_constraint('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('generation_fuel_by_generator_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('generators_eia860', schema=None) as batch_op: - batch_op.drop_constraint('fk_generators_eia860_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_generators_eia860_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('mcoe_generators_yearly', schema=None) as batch_op: - batch_op.drop_constraint('fk_mcoe_generators_yearly_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_mcoe_generators_yearly_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('mega_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_mega_generators_eia_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_mega_generators_eia_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint('fk_ownership_eia860_plant_id_eia_generators_entity_eia', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_ownership_eia860_plant_id_eia_generators_eia860'), 'generators_eia860', ['plant_id_eia', 'generator_id', 'report_date'], ['plant_id_eia', 'generator_id', 'report_date']) - - with op.batch_alter_table('plant_parts_eia', schema=None) as batch_op: - batch_op.drop_constraint('fk_plant_parts_eia_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_plant_parts_eia_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - with op.batch_alter_table('plants_eia860', schema=None) as batch_op: - batch_op.drop_constraint('fk_plants_eia860_utility_id_eia_utilities_eia860', type_='foreignkey') - batch_op.create_foreign_key(batch_op.f('fk_plants_eia860_utility_id_eia_utilities_entity_eia'), 'utilities_entity_eia', ['utility_id_eia'], ['utility_id_eia']) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('plants_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_plants_eia860_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_plants_eia860_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('plant_parts_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_plant_parts_eia_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_plant_parts_eia_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_ownership_eia860_plant_id_eia_generators_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_ownership_eia860_plant_id_eia_generators_entity_eia', 'generators_entity_eia', ['plant_id_eia', 'generator_id'], ['plant_id_eia', 'generator_id']) - - with op.batch_alter_table('mega_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_mega_generators_eia_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_mega_generators_eia_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('mcoe_generators_yearly', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_mcoe_generators_yearly_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_mcoe_generators_yearly_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('generators_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_generators_eia860_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_generators_eia860_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('generation_fuel_by_generator_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_generation_fuel_by_generator_yearly_eia923_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('fuel_cost_by_generator_yearly', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_fuel_cost_by_generator_yearly_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_utilities_eia_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_utilities_eia_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_plants_utilities_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_plants_utilities_eia_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_plants_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_plants_eia_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_plants_eia_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_ownership_eia860', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_ownership_eia860_plant_id_eia_generators_eia860'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_ownership_eia860_plant_id_eia_generators_entity_eia', 'generators_entity_eia', ['plant_id_eia', 'generator_id'], ['plant_id_eia', 'generator_id']) - - with op.batch_alter_table('denorm_generators_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generators_eia_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_generators_eia_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_generation_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_generation_yearly_eia923_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_generation_fuel_combined_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_generation_fuel_combined_yearly_eia923_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_fuel_receipts_costs_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_fuel_receipts_costs_yearly_eia923_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_boilers_eia', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_boilers_eia_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_boilers_eia_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - with op.batch_alter_table('denorm_boiler_fuel_yearly_eia923', schema=None) as batch_op: - batch_op.drop_constraint(batch_op.f('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_entity_eia'), type_='foreignkey') - batch_op.create_foreign_key('fk_denorm_boiler_fuel_yearly_eia923_utility_id_eia_utilities_eia860', 'utilities_eia860', ['utility_id_eia', 'report_date'], ['utility_id_eia', 'report_date']) - - # ### end Alembic commands ### diff --git a/migrations/versions/ad3e7ecdcfc3_add_utility_type.py b/migrations/versions/ad3e7ecdcfc3_add_utility_type.py deleted file mode 100644 index 87288303b5..0000000000 --- a/migrations/versions/ad3e7ecdcfc3_add_utility_type.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Add utility type - -Revision ID: ad3e7ecdcfc3 -Revises: 3313ca078f4e -Create Date: 2023-09-27 15:06:27.671649 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "ad3e7ecdcfc3" -down_revision = "3313ca078f4e" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table("balance_sheet_assets_ferc1", schema=None) as batch_op: - batch_op.add_column( - sa.Column( - "utility_type", - sa.Text(), - nullable=True, - comment="Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.", - ) - ) - - with op.batch_alter_table( - "denorm_balance_sheet_assets_ferc1", schema=None - ) as batch_op: - batch_op.add_column( - sa.Column( - "utility_type", - sa.Text(), - nullable=True, - comment="Listing of utility plant types. Examples include Electric Utility, Gas Utility, and Other Utility.", - ) - ) - - with op.batch_alter_table("denorm_plant_in_service_ferc1", schema=None) as batch_op: - batch_op.add_column( - sa.Column( - "plant_status", - sa.Text(), - nullable=True, - comment="Utility plant financial status (in service, future, leased, total).", - ) - ) - - with op.batch_alter_table("plant_in_service_ferc1", schema=None) as batch_op: - batch_op.add_column( - sa.Column( - "plant_status", - sa.Text(), - nullable=True, - comment="Utility plant financial status (in service, future, leased, total).", - ) - ) - - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table("plant_in_service_ferc1", schema=None) as batch_op: - batch_op.drop_column("plant_status") - - with op.batch_alter_table("denorm_plant_in_service_ferc1", schema=None) as batch_op: - batch_op.drop_column("plant_status") - - with op.batch_alter_table( - "denorm_balance_sheet_assets_ferc1", schema=None - ) as batch_op: - batch_op.drop_column("utility_type") - - with op.batch_alter_table("balance_sheet_assets_ferc1", schema=None) as batch_op: - batch_op.drop_column("utility_type") - - # ### end Alembic commands ### diff --git a/migrations/versions/f11241c9292d_add_my_cool_ferc1_eia_table.py b/migrations/versions/f11241c9292d_add_my_cool_ferc1_eia_table.py deleted file mode 100644 index c69be7a69f..0000000000 --- a/migrations/versions/f11241c9292d_add_my_cool_ferc1_eia_table.py +++ /dev/null @@ -1,799 +0,0 @@ -"""Add my cool ferc1_eia table - -Revision ID: f11241c9292d -Revises: 7fa2763bd630 -Create Date: 2023-10-16 11:02:01.595978 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "f11241c9292d" -down_revision = "7fa2763bd630" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "out__yearly_plants_all_ferc1_plant_parts_eia", - sa.Column( - "record_id_ferc1", - sa.Text(), - nullable=False, - comment="Identifier indicating original FERC Form 1 source record. format: {table_name}_{report_year}_{report_prd}_{respondent_id}_{spplmnt_num}_{row_number}. Unique within FERC Form 1 DB tables which are not row-mapped.", - ), - sa.Column( - "record_id_eia", - sa.Text(), - nullable=True, - comment="Identifier for EIA plant parts analysis records.", - ), - sa.Column( - "match_type", - sa.Text(), - nullable=True, - comment="Indicates the source and validation of the match between EIA and FERC. Match types include matches was generated from the model, verified by the training data, overridden by the training data, etc.", - ), - sa.Column( - "plant_name_ppe", - sa.Text(), - nullable=True, - comment="Derived plant name that includes EIA plant name and other strings associated with ID and PK columns of the plant part.", - ), - sa.Column( - "plant_part", - sa.Enum( - "plant_prime_fuel", - "plant_unit", - "plant_gen", - "plant", - "plant_ferc_acct", - "plant_operating_year", - "plant_technology", - "plant_match_ferc1", - "plant_prime_mover", - ), - nullable=True, - comment="The part of the plant a record corresponds to.", - ), - sa.Column( - "report_year", - sa.Integer(), - nullable=True, - comment="Four-digit year in which the data was reported.", - ), - sa.Column("report_date", sa.Date(), nullable=True, comment="Date reported."), - sa.Column( - "ownership_record_type", - sa.Enum("owned", "total"), - nullable=True, - comment="Whether each generator record is for one owner or represents a total of all ownerships.", - ), - sa.Column("plant_name_eia", sa.Text(), nullable=True, comment="Plant name."), - sa.Column( - "plant_id_eia", - sa.Integer(), - nullable=True, - comment="The unique six-digit facility identification number, also called an ORISPL, assigned by the Energy Information Administration.", - ), - sa.Column( - "generator_id", - sa.Text(), - nullable=True, - comment="Generator ID is usually numeric, but sometimes includes letters. Make sure you treat it as a string!", - ), - sa.Column( - "unit_id_pudl", - sa.Integer(), - nullable=True, - comment="Dynamically assigned PUDL unit id. WARNING: This ID is not guaranteed to be static long term as the input data and algorithm may evolve over time.", - ), - sa.Column( - "prime_mover_code", - sa.Text(), - nullable=True, - comment="Code for the type of prime mover (e.g. CT, CG)", - ), - sa.Column( - "energy_source_code_1", - sa.Text(), - nullable=True, - comment="The code representing the most predominant type of energy that fuels the generator.", - ), - sa.Column( - "technology_description", - sa.Text(), - nullable=True, - comment="High level description of the technology used by the generator to produce electricity.", - ), - sa.Column( - "ferc_acct_name", - sa.Enum("Hydraulic", "Nuclear", "Steam", "Other"), - nullable=True, - comment="Name of FERC account, derived from technology description and prime mover code.", - ), - sa.Column( - "generator_operating_year", - sa.Integer(), - nullable=True, - comment="Year a generator went into service.", - ), - sa.Column( - "utility_id_eia", - sa.Integer(), - nullable=True, - comment="The EIA Utility Identification number.", - ), - sa.Column( - "utility_id_pudl", - sa.Integer(), - nullable=True, - comment="A manually assigned PUDL utility ID. May not be stable over time.", - ), - sa.Column( - "true_gran", - sa.Boolean(), - nullable=True, - comment="Indicates whether a plant part list record is associated with the highest priority plant part for all identical records.", - ), - sa.Column( - "appro_part_label", - sa.Enum( - "plant_prime_fuel", - "plant_unit", - "plant_gen", - "plant", - "plant_ferc_acct", - "plant_operating_year", - "plant_technology", - "plant_match_ferc1", - "plant_prime_mover", - ), - nullable=True, - comment="Plant part of the associated true granularity record.", - ), - sa.Column( - "appro_record_id_eia", - sa.Text(), - nullable=True, - comment="EIA record ID of the associated true granularity record.", - ), - sa.Column( - "record_count", - sa.Integer(), - nullable=True, - comment="Number of distinct generator IDs that partcipated in the aggregation for a plant part list record.", - ), - sa.Column( - "fraction_owned", - sa.Float(), - nullable=True, - comment="Proportion of generator ownership attributable to this utility.", - ), - sa.Column( - "ownership_dupe", - sa.Boolean(), - nullable=True, - comment="Whether a plant part record has a duplicate record with different ownership status.", - ), - sa.Column( - "operational_status", - sa.Text(), - nullable=True, - comment="The operating status of the asset. For generators this is based on which tab the generator was listed in in EIA 860.", - ), - sa.Column( - "operational_status_pudl", - sa.Enum("operating", "retired", "proposed"), - nullable=True, - comment="The operating status of the asset using PUDL categories.", - ), - sa.Column( - "plant_id_pudl", - sa.Integer(), - nullable=True, - comment="A manually assigned PUDL plant ID. May not be constant over time.", - ), - sa.Column( - "total_fuel_cost_eia", - sa.Float(), - nullable=True, - comment="Total annual reported fuel costs for the plant part. Includes costs from all fuels.", - ), - sa.Column( - "fuel_cost_per_mmbtu_eia", - sa.Float(), - nullable=True, - comment="Average fuel cost per mmBTU of heat content in nominal USD.", - ), - sa.Column( - "net_generation_mwh_eia", - sa.Float(), - nullable=True, - comment="Net electricity generation for the specified period in megawatt-hours (MWh).", - ), - sa.Column( - "capacity_mw_eia", - sa.Float(), - nullable=True, - comment="Total installed (nameplate) capacity, in megawatts.", - ), - sa.Column( - "capacity_factor_eia", - sa.Float(), - nullable=True, - comment="Fraction of potential generation that was actually reported for a plant part.", - ), - sa.Column( - "total_mmbtu_eia", - sa.Float(), - nullable=True, - comment="Total annual heat content of fuel consumed by a plant part record in the plant parts list.", - ), - sa.Column( - "heat_rate_mmbtu_mwh_eia", - sa.Float(), - nullable=True, - comment="Fuel content per unit of electricity generated. Coming from MCOE calculation.", - ), - sa.Column( - "fuel_type_code_pudl_eia", - sa.Enum( - "coal", - "gas", - "hydro", - "nuclear", - "oil", - "other", - "solar", - "waste", - "wind", - ), - nullable=True, - comment="Simplified fuel type code used in PUDL", - ), - sa.Column( - "installation_year_eia", - sa.Integer(), - nullable=True, - comment="Year the plant's most recently built unit was installed.", - ), - sa.Column( - "plant_part_id_eia", - sa.Text(), - nullable=True, - comment="Contains EIA plant ID, plant part, ownership, and EIA utility id", - ), - sa.Column( - "utility_id_ferc1", - sa.Integer(), - nullable=True, - comment="PUDL-assigned utility ID, identifying a FERC1 utility. This is an auto-incremented ID and is not expected to be stable from year to year.", - ), - sa.Column( - "utility_name_ferc1", - sa.Text(), - nullable=True, - comment="Name of the responding utility, as it is reported in FERC Form 1. For human readability only.", - ), - sa.Column( - "plant_id_ferc1", - sa.Integer(), - nullable=True, - comment="Algorithmically assigned PUDL FERC Plant ID. WARNING: NOT STABLE BETWEEN PUDL DB INITIALIZATIONS.", - ), - sa.Column( - "plant_name_ferc1", - sa.Text(), - nullable=True, - comment="Name of the plant, as reported to FERC. This is a freeform string, not guaranteed to be consistent across references to the same plant.", - ), - sa.Column( - "asset_retirement_cost", - sa.Float(), - nullable=True, - comment="Asset retirement cost (USD).", - ), - sa.Column("avg_num_employees", sa.Float(), nullable=True), - sa.Column( - "capacity_factor_ferc1", - sa.Float(), - nullable=True, - comment="Fraction of potential generation that was actually reported for a plant part.", - ), - sa.Column( - "capacity_mw_ferc1", - sa.Float(), - nullable=True, - comment="Total installed (nameplate) capacity, in megawatts.", - ), - sa.Column( - "capex_annual_addition", - sa.Float(), - nullable=True, - comment="Annual capital addition into `capex_total`.", - ), - sa.Column( - "capex_annual_addition_rolling", - sa.Float(), - nullable=True, - comment="Year-to-date capital addition into `capex_total`.", - ), - sa.Column( - "capex_annual_per_kw", - sa.Float(), - nullable=True, - comment="Annual capital addition into `capex_total` per kw.", - ), - sa.Column( - "capex_annual_per_mw", - sa.Float(), - nullable=True, - comment="Annual capital addition into `capex_total` per MW.", - ), - sa.Column( - "capex_annual_per_mw_rolling", - sa.Float(), - nullable=True, - comment="Year-to-date capital addition into `capex_total` per MW.", - ), - sa.Column( - "capex_annual_per_mwh", - sa.Float(), - nullable=True, - comment="Annual capital addition into `capex_total` per MWh.", - ), - sa.Column( - "capex_annual_per_mwh_rolling", - sa.Float(), - nullable=True, - comment="Year-to-date capital addition into `capex_total` per MWh.", - ), - sa.Column( - "capex_equipment", - sa.Float(), - nullable=True, - comment="Cost of plant: equipment (USD).", - ), - sa.Column( - "capex_land", - sa.Float(), - nullable=True, - comment="Cost of plant: land and land rights (USD).", - ), - sa.Column( - "capex_per_mw", - sa.Float(), - nullable=True, - comment="Cost of plant per megawatt of installed (nameplate) capacity. Nominal USD.", - ), - sa.Column( - "capex_structures", - sa.Float(), - nullable=True, - comment="Cost of plant: structures and improvements (USD).", - ), - sa.Column( - "capex_total", - sa.Float(), - nullable=True, - comment="Total cost of plant (USD).", - ), - sa.Column( - "capex_wo_retirement_total", - sa.Float(), - nullable=True, - comment="Total cost of plant (USD) without retirements.", - ), - sa.Column( - "construction_type", - sa.Enum("conventional", "outdoor", "semioutdoor"), - nullable=True, - comment="Type of plant construction ('outdoor', 'semioutdoor', or 'conventional'). Categorized by PUDL based on our best guess of intended value in FERC1 freeform strings.", - ), - sa.Column( - "construction_year_eia", - sa.Integer(), - nullable=True, - comment="Year the plant's oldest still operational unit was built.", - ), - sa.Column( - "construction_year_ferc1", - sa.Integer(), - nullable=True, - comment="Year the plant's oldest still operational unit was built.", - ), - sa.Column( - "installation_year_ferc1", - sa.Integer(), - nullable=True, - comment="Year the plant's most recently built unit was installed.", - ), - sa.Column( - "net_generation_mwh_ferc1", - sa.Float(), - nullable=True, - comment="Net electricity generation for the specified period in megawatt-hours (MWh).", - ), - sa.Column( - "not_water_limited_capacity_mw", - sa.Float(), - nullable=True, - comment="Plant capacity in MW when not limited by condenser water.", - ), - sa.Column("opex_allowances", sa.Float(), nullable=True, comment="Allowances."), - sa.Column( - "opex_boiler", - sa.Float(), - nullable=True, - comment="Maintenance of boiler (or reactor) plant.", - ), - sa.Column( - "opex_coolants", - sa.Float(), - nullable=True, - comment="Cost of coolants and water (nuclear plants only)", - ), - sa.Column( - "opex_electric", - sa.Float(), - nullable=True, - comment="Production expenses: electric expenses (USD).", - ), - sa.Column( - "opex_engineering", - sa.Float(), - nullable=True, - comment="Production expenses: maintenance, supervision, and engineering (USD).", - ), - sa.Column( - "opex_fuel", - sa.Float(), - nullable=True, - comment="Production expenses: fuel (USD).", - ), - sa.Column( - "fuel_cost_per_mwh_eia", - sa.Float(), - nullable=True, - comment="Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.", - ), - sa.Column( - "fuel_cost_per_mwh_ferc1", - sa.Float(), - nullable=True, - comment="Derived from MCOE, a unit level value. Average fuel cost per MWh of heat content in nominal USD.", - ), - sa.Column( - "opex_misc_power", - sa.Float(), - nullable=True, - comment="Miscellaneous steam (or nuclear) expenses.", - ), - sa.Column( - "opex_misc_steam", - sa.Float(), - nullable=True, - comment="Maintenance of miscellaneous steam (or nuclear) plant.", - ), - sa.Column( - "opex_nonfuel_per_mwh", - sa.Float(), - nullable=True, - comment="Investments in non-fuel production expenses per Mwh.", - ), - sa.Column( - "opex_operations", - sa.Float(), - nullable=True, - comment="Production expenses: operations, supervision, and engineering (USD).", - ), - sa.Column( - "opex_per_mwh", - sa.Float(), - nullable=True, - comment="Total production expenses (USD per MWh generated).", - ), - sa.Column( - "opex_plant", - sa.Float(), - nullable=True, - comment="Production expenses: maintenance of electric plant (USD).", - ), - sa.Column( - "opex_production_total", - sa.Float(), - nullable=True, - comment="Total operating expenses.", - ), - sa.Column( - "opex_rents", - sa.Float(), - nullable=True, - comment="Production expenses: rents (USD).", - ), - sa.Column("opex_steam", sa.Float(), nullable=True, comment="Steam expenses."), - sa.Column( - "opex_steam_other", - sa.Float(), - nullable=True, - comment="Steam from other sources.", - ), - sa.Column( - "opex_structures", - sa.Float(), - nullable=True, - comment="Production expenses: maintenance of structures (USD).", - ), - sa.Column( - "opex_total_nonfuel", - sa.Float(), - nullable=True, - comment="Total production expenses, excluding fuel (USD).", - ), - sa.Column( - "opex_transfer", - sa.Float(), - nullable=True, - comment="Steam transferred (Credit).", - ), - sa.Column( - "peak_demand_mw", - sa.Float(), - nullable=True, - comment="Net peak demand for 60 minutes. Note: in some cases peak demand for other time periods may have been reported instead, if hourly peak demand was unavailable.", - ), - sa.Column( - "plant_capability_mw", - sa.Float(), - nullable=True, - comment="Net plant capability in megawatts.", - ), - sa.Column( - "plant_hours_connected_while_generating", - sa.Float(), - nullable=True, - comment="Hours the plant was connected to load while generating in the report year.", - ), - sa.Column("plant_type", sa.Text(), nullable=True), - sa.Column( - "water_limited_capacity_mw", - sa.Float(), - nullable=True, - comment="Plant capacity in MW when limited by condenser water.", - ), - sa.Column( - "fuel_cost_per_mmbtu_ferc1", - sa.Float(), - nullable=True, - comment="Average fuel cost per mmBTU of heat content in nominal USD.", - ), - sa.Column("fuel_type", sa.Text(), nullable=True), - sa.Column( - "license_id_ferc1", - sa.Integer(), - nullable=True, - comment="FERC issued operating license ID for the facility, if available. This value is extracted from the original plant name where possible.", - ), - sa.Column( - "opex_maintenance", - sa.Float(), - nullable=True, - comment="Production expenses: Maintenance (USD).", - ), - sa.Column( - "opex_total", - sa.Float(), - nullable=True, - comment="Total production expenses, excluding fuel (USD).", - ), - sa.Column( - "capex_facilities", - sa.Float(), - nullable=True, - comment="Cost of plant: reservoirs, dams, and waterways (USD).", - ), - sa.Column( - "capex_roads", - sa.Float(), - nullable=True, - comment="Cost of plant: roads, railroads, and bridges (USD).", - ), - sa.Column( - "net_capacity_adverse_conditions_mw", - sa.Float(), - nullable=True, - comment="Net plant capability under the least favorable operating conditions, in megawatts.", - ), - sa.Column( - "net_capacity_favorable_conditions_mw", - sa.Float(), - nullable=True, - comment="Net plant capability under the most favorable operating conditions, in megawatts.", - ), - sa.Column( - "opex_dams", - sa.Float(), - nullable=True, - comment="Production expenses: maintenance of reservoirs, dams, and waterways (USD).", - ), - sa.Column( - "opex_generation_misc", - sa.Float(), - nullable=True, - comment="Production expenses: miscellaneous power generation expenses (USD).", - ), - sa.Column( - "opex_hydraulic", - sa.Float(), - nullable=True, - comment="Production expenses: hydraulic expenses (USD).", - ), - sa.Column( - "opex_misc_plant", - sa.Float(), - nullable=True, - comment="Production expenses: maintenance of miscellaneous hydraulic plant (USD).", - ), - sa.Column( - "opex_water_for_power", - sa.Float(), - nullable=True, - comment="Production expenses: water for power (USD).", - ), - sa.Column( - "ferc_license_id", - sa.Text(), - nullable=True, - comment="The FERC license ID of a project.", - ), - sa.Column( - "capex_equipment_electric", - sa.Float(), - nullable=True, - comment="Cost of plant: accessory electric equipment (USD).", - ), - sa.Column( - "capex_equipment_misc", - sa.Float(), - nullable=True, - comment="Cost of plant: miscellaneous power plant equipment (USD).", - ), - sa.Column( - "capex_wheels_turbines_generators", - sa.Float(), - nullable=True, - comment="Cost of plant: water wheels, turbines, and generators (USD).", - ), - sa.Column( - "energy_used_for_pumping_mwh", - sa.Float(), - nullable=True, - comment="Energy used for pumping, in megawatt-hours.", - ), - sa.Column( - "net_load_mwh", - sa.Float(), - nullable=True, - comment="Net output for load (net generation - energy used for pumping) in megawatt-hours.", - ), - sa.Column( - "opex_production_before_pumping", - sa.Float(), - nullable=True, - comment="Total production expenses before pumping (USD).", - ), - sa.Column( - "opex_pumped_storage", - sa.Float(), - nullable=True, - comment="Production expenses: pumped storage (USD).", - ), - sa.Column( - "opex_pumping", - sa.Float(), - nullable=True, - comment="Production expenses: We are here to PUMP YOU UP! (USD).", - ), - sa.Column( - "total_fuel_cost_ferc1", - sa.Float(), - nullable=True, - comment="Total annual reported fuel costs for the plant part. Includes costs from all fuels.", - ), - sa.Column( - "total_mmbtu_ferc1", - sa.Float(), - nullable=True, - comment="Total annual heat content of fuel consumed by a plant part record in the plant parts list.", - ), - sa.Column( - "fuel_type_code_pudl_ferc1", - sa.Enum( - "coal", - "gas", - "hydro", - "nuclear", - "oil", - "other", - "solar", - "waste", - "wind", - ), - nullable=True, - comment="Simplified fuel type code used in PUDL", - ), - sa.Column( - "heat_rate_mmbtu_mwh_ferc1", - sa.Float(), - nullable=True, - comment="Fuel content per unit of electricity generated. Calculated from FERC reported fuel consumption and net generation.", - ), - sa.ForeignKeyConstraint( - ["energy_source_code_1"], - ["energy_sources_eia.code"], - name=op.f( - "fk_out__yearly_plants_all_ferc1_plant_parts_eia_energy_source_code_1_energy_sources_eia" - ), - ), - sa.ForeignKeyConstraint( - ["plant_id_eia", "generator_id", "report_date"], - [ - "generators_eia860.plant_id_eia", - "generators_eia860.generator_id", - "generators_eia860.report_date", - ], - name=op.f( - "fk_out__yearly_plants_all_ferc1_plant_parts_eia_plant_id_eia_generators_eia860" - ), - ), - sa.ForeignKeyConstraint( - ["plant_id_pudl"], - ["plants_pudl.plant_id_pudl"], - name=op.f( - "fk_out__yearly_plants_all_ferc1_plant_parts_eia_plant_id_pudl_plants_pudl" - ), - ), - sa.ForeignKeyConstraint( - ["prime_mover_code"], - ["prime_movers_eia.code"], - name=op.f( - "fk_out__yearly_plants_all_ferc1_plant_parts_eia_prime_mover_code_prime_movers_eia" - ), - ), - sa.ForeignKeyConstraint( - ["utility_id_eia", "report_date"], - ["utilities_eia860.utility_id_eia", "utilities_eia860.report_date"], - name=op.f( - "fk_out__yearly_plants_all_ferc1_plant_parts_eia_utility_id_eia_utilities_eia860" - ), - ), - sa.ForeignKeyConstraint( - ["utility_id_ferc1", "plant_name_ferc1"], - ["plants_ferc1.utility_id_ferc1", "plants_ferc1.plant_name_ferc1"], - name=op.f( - "fk_out__yearly_plants_all_ferc1_plant_parts_eia_utility_id_ferc1_plants_ferc1" - ), - ), - sa.ForeignKeyConstraint( - ["utility_id_pudl"], - ["utilities_pudl.utility_id_pudl"], - name=op.f( - "fk_out__yearly_plants_all_ferc1_plant_parts_eia_utility_id_pudl_utilities_pudl" - ), - ), - sa.PrimaryKeyConstraint( - "record_id_ferc1", - name=op.f("pk_out__yearly_plants_all_ferc1_plant_parts_eia"), - ), - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("out__yearly_plants_all_ferc1_plant_parts_eia") - # ### end Alembic commands ### diff --git a/notebooks/work-in-progress/epacems-parquet.ipynb b/notebooks/work-in-progress/epacems-parquet.ipynb index df5a1a0268..5ab1c67584 100644 --- a/notebooks/work-in-progress/epacems-parquet.ipynb +++ b/notebooks/work-in-progress/epacems-parquet.ipynb @@ -1427,53 +1427,35 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 8, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1 s, sys: 14.4 ms, total: 1.02 s\n", - "Wall time: 1.02 s\n" - ] - } - ], + "outputs": [], "source": [ - "\n", - "ops_cols = [\n", - " 'co2_mass_tons',\n", - " 'facility_id',\n", - " 'gross_load_mw',\n", - " 'heat_content_mmbtu',\n", - " 'operating_time_hours',\n", - " 'plant_id_eia',\n", - " 'state',\n", - " 'unit_id_epa',\n", - " 'unitid',\n", - " 'operating_datetime'\n", - "]\n", - "epacems_datadir = os.path.join(pudl.settings.PUDL_DIR,'results','parquet','epacems')\n", - "cems_dd = dd.read_parquet(epacems_datadir + '/*/*.parquet', columns=ops_cols)" + "import os\n", + "import dask.dataframe as dd\n", + "import pudl\n", + "#epacems_datadir = os.path.join(,'pudl_output','hourly_emissions_epacems')\n", + "cems_dd = dd.read_parquet(\"/home/thinky/Desktop/Catalyst/PUDL/pudl-work/pudl_output/hourly_emissions_epacems\" + '*.parquet')" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 10, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Columns: 10 entries, co2_mass_tons to operating_datetime\n", - "dtypes: category(4), datetime64[ns](1), float32(4), uint16(1)" - ] + "data": { + "text/plain": [ + "Index(['plant_id_eia', 'plant_id_epa', 'emissions_unit_id_epa', 'operating_datetime_utc', 'year', 'state', 'operating_time_hours', 'gross_load_mw', 'heat_content_mmbtu', 'steam_load_1000_lbs', 'so2_mass_lbs', 'so2_mass_measurement_code', 'nox_mass_lbs', 'nox_mass_measurement_code', 'co2_mass_tons', 'co2_mass_measurement_code'], dtype='object')" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "cems_dd.info()" + "cems_dd.columns" ] }, { @@ -2273,7 +2255,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.3" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/src/pudl/analysis/__init__.py b/src/pudl/analysis/__init__.py index 3b489e3d8e..44b2019e7a 100644 --- a/src/pudl/analysis/__init__.py +++ b/src/pudl/analysis/__init__.py @@ -7,8 +7,8 @@ """ from . import ( allocate_gen_fuel, + eia_ferc1_record_linkage, epacamd_eia, - ferc1_eia_record_linkage, mcoe, plant_parts_eia, service_territory, diff --git a/src/pudl/analysis/allocate_gen_fuel.py b/src/pudl/analysis/allocate_gen_fuel.py index bf7a71bedb..8ca8396f69 100644 --- a/src/pudl/analysis/allocate_gen_fuel.py +++ b/src/pudl/analysis/allocate_gen_fuel.py @@ -1,32 +1,32 @@ -"""Allocate data from :ref:`generation_fuel_eia923` table to generator level. +"""Allocate data from :ref:`core_eia923__monthly_generation_fuel` table to generator level. The algorithm we're using assumes the following about the reported data: -* The :ref:`generation_fuel_eia923` table is the authoritative source of information +* The :ref:`core_eia923__monthly_generation_fuel` table is the authoritative source of information about how much generation and fuel consumption is attributable to an entire plant. This table has the most complete data coverage, but it is not the most granular data reported. It's primary keys are :py:const:`IDX_PM_ESC`. -* The :ref:`generation_eia923` table contains the most granular net generation data. It +* The :ref:`core_eia923__monthly_generation` table contains the most granular net generation data. It is reported at the generator level with primary keys :py:const:`IDX_GENS`. This table - includes only ~39% of the total MWhs reported in the :ref:`generation_fuel_eia923` + includes only ~39% of the total MWhs reported in the :ref:`core_eia923__monthly_generation_fuel` table. -* The :ref:`boiler_fuel_eia923` table contains the most granular fuel consumption data. +* The :ref:`core_eia923__monthly_boiler_fuel` table contains the most granular fuel consumption data. It is reported at the boiler/prime mover/energy source level with primary keys :py:const:`IDX_B_PM_ESC`. This table includes only ~38% of the total MMBTUs reported - in the :ref:`generation_fuel_eia923` table. -* The :ref:`generators_eia860` table provides an exhaustive list of all generators - whose generation is being reported in the :ref:`generation_fuel_eia923` table - with + in the :ref:`core_eia923__monthly_generation_fuel` table. +* The :ref:`core_eia860__scd_generators` table provides an exhaustive list of all generators + whose generation is being reported in the :ref:`core_eia923__monthly_generation_fuel` table - with primary keys :py:const:`IDX_GENS`. This module allocates the total net electricity generation and fuel consumption -reported in the :ref:`generation_fuel_eia923` table to individual generators, based -on more granular data reported in the :ref:`generation_eia923` and -:ref:`boiler_fuel_eia923` tables, as well as capacity (MW) found in the -:ref:`generators_eia860` table. It uses other generator attributes from the -:ref:`generators_eia860` table to associate the data found in the -:ref:`generation_fuel_eia923` with generators. It also uses as the associations between -boilers and generators found in the :ref:`boiler_generator_assn_eia860` table to -aggregate data :ref:`boiler_fuel_eia923` tables. The main coordinating functions hereare +reported in the :ref:`core_eia923__monthly_generation_fuel` table to individual generators, based +on more granular data reported in the :ref:`core_eia923__monthly_generation` and +:ref:`core_eia923__monthly_boiler_fuel` tables, as well as capacity (MW) found in the +:ref:`core_eia860__scd_generators` table. It uses other generator attributes from the +:ref:`core_eia860__scd_generators` table to associate the data found in the +:ref:`core_eia923__monthly_generation_fuel` with generators. It also uses as the associations between +boilers and generators found in the :ref:`core_eia860__assn_boiler_generator` table to +aggregate data :ref:`core_eia923__monthly_boiler_fuel` tables. The main coordinating functions hereare :func:`allocate_gen_fuel_by_generator_energy_source` and :func:`aggregate_gen_fuel_by_generator`. @@ -34,8 +34,8 @@ * **Data columns** refers to the net generation and fuel consumption - the specific columns are defined in :py:const:`DATA_COLUMNS`. -* **Granular tables** refers to :ref:`generation_eia923` and - :ref:`boiler_fuel_eia923`, which report granular data but do not have complete +* **Granular tables** refers to :ref:`core_eia923__monthly_generation` and + :ref:`core_eia923__monthly_boiler_fuel`, which report granular data but do not have complete coverage. There are six main stages of the allocation process in this module: @@ -52,9 +52,9 @@ record are directly reported in the granular tables. This lets us choose an appropriate data allocation method based on how complete the granular data coverage is for a given value of :py:const:`IDX_PM_ESC`, which is the original primary key of - the :ref:`generation_fuel_eia923` table. (See :func:`prep_allocation_fraction`). + the :ref:`core_eia923__monthly_generation_fuel` table. (See :func:`prep_allocation_fraction`). #. **Allocate**: Allocate the net generation and fuel consumption reported in the less - granular :ref:`generation_fuel_eia923` table to the :py:const:`IDX_GENS_PM_ESC` + granular :ref:`core_eia923__monthly_generation_fuel` table to the :py:const:`IDX_GENS_PM_ESC` level. More details on the allocation process are below (see :func:`allocate_gen_fuel_by_gen_esc` and :func:`allocate_fuel_by_gen_esc`). #. **Sanity check allocation**: Verify that the total allocated net generation and fuel @@ -69,7 +69,7 @@ **High-level description about the allocaiton step**: -We allocate the data columns reported in the :ref:`generation_fuel_eia923` table on the +We allocate the data columns reported in the :ref:`core_eia923__monthly_generation_fuel` table on the basis of plant, prime mover, and energy source among the generators in each plant that have matching energy sources. @@ -89,18 +89,18 @@ tables. In the **ALL** generators case, the data columns reported in the -:ref:`generation_fuel_eia923` table are allocated in proportion to data reported in the +:ref:`core_eia923__monthly_generation_fuel` table are allocated in proportion to data reported in the granular data tables. We do this instead of directly using the data columns from the -granular tables because there are discrepancies between the generation_fuel_eia923 table +granular tables because there are discrepancies between the core_eia923__monthly_generation_fuel table and the granular tables and we are assuming the totals reported in the -generation_fuel_eia923 table are authoritative. +core_eia923__monthly_generation_fuel table are authoritative. In the **NONE** generators case, the data columns reported in the -:ref:`generation_fuel_eia923` table are allocated in proportion to the each generator's +:ref:`core_eia923__monthly_generation_fuel` table are allocated in proportion to the each generator's capacity. In the **SOME** generators case, we use a combination of the two allocation methods -described above. First, the data columns reported in the :ref:`generation_fuel_eia923` +described above. First, the data columns reported in the :ref:`core_eia923__monthly_generation_fuel` table are allocated between the two categories of generators: those that report granular data, and those that don't. The fraction allocated to each of those categories is based on how much of the total is reported in the granular tables. If T is the total reported, @@ -123,7 +123,7 @@ equally among them. In effect, if a plant had multiple generators with the same prime_mover_code but opposite primary and secondary fuels (eg. gen 1 has a primary fuel of 'NG' and secondary fuel of 'DFO', while gen 2 has a primary fuel of 'DFO' and a -secondary fuel of 'NG'), the methodology associates the generation_fuel_eia923 records +secondary fuel of 'NG'), the methodology associates the core_eia923__monthly_generation_fuel records similarly across these two generators. However, the allocated net generation will still be porporational to each generator's net generation (if it's reported) or capacity (if generation is not reported). @@ -177,7 +177,7 @@ "fuel_consumed_mmbtu", "fuel_consumed_for_electricity_mmbtu", ] -"""Data columns from :ref:`generation_fuel_eia923` that are being allocated.""" +"""Data columns from :ref:`core_eia923__monthly_generation_fuel` that are being allocated.""" MISSING_SENTINEL = 0.00001 """A sentinel value for dealing with null or zero values. @@ -208,15 +208,15 @@ def allocate_gen_fuel_asset_factory( raise ValueError(f"freq must be one of {agg_freqs.keys()}, got: {freq}.") @asset( - name=f"generation_fuel_by_generator_energy_source_{agg_freqs[freq]}_eia923", + name=f"out_eia923__{agg_freqs[freq]}_generation_fuel_by_generator_energy_source", ins={ "gf": AssetIn( - key=f"denorm_generation_fuel_combined_{agg_freqs[freq]}_eia923" + key=f"out_eia923__{agg_freqs[freq]}_generation_fuel_combined" ), - "bf": AssetIn(key=f"denorm_boiler_fuel_{agg_freqs[freq]}_eia923"), - "gen": AssetIn(key=f"denorm_generation_{agg_freqs[freq]}_eia923"), - "bga": AssetIn(key="boiler_generator_assn_eia860"), - "gens": AssetIn(key="denorm_generators_eia"), + "bf": AssetIn(key=f"out_eia923__{agg_freqs[freq]}_boiler_fuel"), + "gen": AssetIn(key=f"out_eia923__{agg_freqs[freq]}_generation"), + "bga": AssetIn(key="core_eia860__assn_boiler_generator"), + "gens": AssetIn(key="_out_eia__yearly_generators"), }, io_manager_key=io_manager_key, compute_kind="Python", @@ -256,13 +256,13 @@ def gen_fuel_by_gen_esc( ) @asset( - name=f"generation_fuel_by_generator_{agg_freqs[freq]}_eia923", + name=f"out_eia923__{agg_freqs[freq]}_generation_fuel_by_generator", ins={ "net_gen_fuel_alloc": AssetIn( - key=f"generation_fuel_by_generator_energy_source_{agg_freqs[freq]}_eia923" + key=f"out_eia923__{agg_freqs[freq]}_generation_fuel_by_generator_energy_source" ), - "pu": AssetIn(key="denorm_plants_utilities_eia"), - "bga": AssetIn(key="boiler_generator_assn_eia860"), + "pu": AssetIn(key="_out_eia__plants_utilities"), + "bga": AssetIn(key="core_eia860__assn_boiler_generator"), }, io_manager_key=io_manager_key, compute_kind="Python", @@ -277,18 +277,18 @@ def gen_fuel_by_gen( net_gen_fuel_alloc=net_gen_fuel_alloc, sum_cols=DATA_COLUMNS, ) - # make the output resemble denorm_generation_eia923: + # make the output resemble out_eia923__generation: .pipe(pudl.output.eia923.denorm_by_gen, pu=pu, bga=bga) ) @asset( - name=f"generation_fuel_by_generator_energy_source_owner_{agg_freqs[freq]}_eia923", + name=f"out_eia923__{agg_freqs[freq]}_generation_fuel_by_generator_energy_source_owner", ins={ "net_gen_fuel_alloc": AssetIn( - key=f"generation_fuel_by_generator_energy_source_{agg_freqs[freq]}_eia923" + key=f"out_eia923__{agg_freqs[freq]}_generation_fuel_by_generator_energy_source" ), - "gens": AssetIn(key="denorm_generators_eia"), - "own_eia860": AssetIn(key="denorm_ownership_eia860"), + "gens": AssetIn(key="_out_eia__yearly_generators"), + "own_eia860": AssetIn(key="out_eia860__yearly_ownership"), }, io_manager_key=io_manager_key, compute_kind="Python", @@ -334,25 +334,25 @@ def allocate_gen_fuel_by_generator_energy_source( There are two main steps here: - * associate ``generation_fuel_eia923`` table data w/ generators - * allocate ``generation_fuel_eia923`` table data proportionally + * associate ``core_eia923__monthly_generation_fuel`` table data w/ generators + * allocate ``core_eia923__monthly_generation_fuel`` table data proportionally The association process happens via :func:`associate_generator_tables`. The allocation process (via :func:`allocate_gen_fuel_by_gen_esc`) entails generating a fraction for each record within a ``IDX_PM_ESC`` group. We have two data points for generating this ratio: the net generation in the - generation_eia923 table and the capacity from the generators_eia860 table. + core_eia923__monthly_generation table and the capacity from the core_eia860__scd_generators table. The end result is a ``frac`` column which is unique for each combination of generator, prime_mover, and fuel and is used to allocate the associated - net generation from the :ref:`generation_fuel_eia923` table. + net generation from the :ref:`core_eia923__monthly_generation_fuel` table. Args: - gf: Temporally aggregated :ref:`denorm_generation_fuel_combined_eia923` dataframe. - bf: Temporally aggregated :ref:`boiler_fuel_eia923` dataframe. - gen: Temporally aggregated :ref:`generation_eia923` dataframe. - bga: :ref:`boiler_generator_assn_eia860` dataframe. - gens: :ref:`generators_eia860` dataframe. + gf: Temporally aggregated :ref:`out_eia923__generation_fuel_combined` dataframe. + bf: Temporally aggregated :ref:`core_eia923__monthly_boiler_fuel` dataframe. + gen: Temporally aggregated :ref:`core_eia923__monthly_generation` dataframe. + bga: :ref:`core_eia860__assn_boiler_generator` dataframe. + gens: :ref:`core_eia860__scd_generators` dataframe. freq: Frequency at which the tables are aggregated temporally. debug: If True, return additional debugging information. """ @@ -478,7 +478,7 @@ def select_input_data( logger.info( f"The granular data tables contain {granular_fuel_ratio:.1%} of the fuel " f"and {granular_net_gen_ratio:.1%} of net generation in the " - "higher-coverage generation_fuel_eia923 table." + "higher-coverage core_eia923__monthly_generation_fuel table." ) return gf, bf, gen, bga, gens @@ -494,9 +494,9 @@ def standardize_input_frequency( table has all of the generators present. Args: - bf: :ref:`boiler_fuel_eia923` table - gens: :ref:`generators_eia860` table - gen: :ref:`generation_eia923` table + bf: :ref:`core_eia923__monthly_boiler_fuel` table + gens: :ref:`core_eia860__scd_generators` table + gen: :ref:`core_eia923__monthly_generation` table freq: the (time) frequency at which the tables will be aggregated. """ bf = distribute_annually_reported_data_to_months_if_annual( @@ -558,9 +558,9 @@ def scale_allocated_net_gen_fuel_by_ownership( net_gen_fuel_alloc: table of allocated generation and fuel consumption at the generator, prime mover, and energy source. From :func:`allocate_gen_fuel_by_generator_energy_source` - gens: ``generators_eia860`` table with cols: :const:``IDX_GENS``, + gens: ``core_eia860__scd_generators`` table with cols: :const:``IDX_GENS``, ``capacity_mw`` and ``utility_id_eia`` - own_eia860: ``ownership_eia860`` table. + own_eia860: ``core_eia860__scd_ownership`` table. """ return pudl.helpers.scale_by_ownership( gens=pudl.helpers.date_merge( @@ -612,7 +612,7 @@ def stack_generators( """Stack the generator table with a set of columns. Args: - gens: generators_eia860 table with cols: :py:const:`IDX_GENS` and all of the + gens: core_eia860__scd_generators table with cols: :py:const:`IDX_GENS` and all of the ``energy_source_code`` columns cat_col: name of category column which will end up having the column names of ``cols_to_stack`` @@ -662,12 +662,12 @@ def associate_generator_tables( ) -> pd.DataFrame: """Associate the three tables needed to assign net gen and fuel to generators. - The :ref:`generation_fuel_eia923` table's data is reported at the - :py:const:`IDX_PM_ESC` granularity. Each generator in the :ref:`generators_eia860` + The :ref:`core_eia923__monthly_generation_fuel` table's data is reported at the + :py:const:`IDX_PM_ESC` granularity. Each generator in the :ref:`core_eia860__scd_generators` has one ``prime_mover_code``, but potentially several ``energy_source_code``s that are reported in several columns. We need to reshape the generators table such that each generator has a separate record corresponding to each of its reported - energy_source_codes, so it can be merged with the :ref:`generation_fuel_eia923` + energy_source_codes, so it can be merged with the :ref:`core_eia923__monthly_generation_fuel` table. We do this using :func:``stack_generators`` employing :func:`pd.DataFrame.stack`. @@ -689,24 +689,24 @@ def associate_generator_tables( more details. There are some records in the data tables that have either ``prime_mover_code`` s or - ``energy_source_code`` s that do no appear in the :ref:`generators_eia860` table. + ``energy_source_code`` s that do no appear in the :ref:`core_eia860__scd_generators` table. We employ :func:`_allocate_unassociated_bf_records` to make sure those records are associated. Args: - gens: :ref:`generators_eia860` table with cols: :py:const:`IDX_GENS` and all of + gens: :ref:`core_eia860__scd_generators` table with cols: :py:const:`IDX_GENS` and all of the ``energy_source_code`` columns and expanded to the same frequency. - gf: :ref:`generation_fuel_eia923` table with columns: :py:const:`IDX_PM_ESC` and + gf: :ref:`core_eia923__monthly_generation_fuel` table with columns: :py:const:`IDX_PM_ESC` and ``net_generation_mwh`` and ``fuel_consumed_mmbtu``. - gen: :ref:`generation_eia923` table with columns: :py:const:`IDX_GENS` and + gen: :ref:`core_eia923__monthly_generation` table with columns: :py:const:`IDX_GENS` and ``net_generation_mwh``. - bf: :ref:`boiler_fuel_eia923` table with columns: :py:const:`IDX_B_PM_ESC` and + bf: :ref:`core_eia923__monthly_boiler_fuel` table with columns: :py:const:`IDX_B_PM_ESC` and fuel consumption columns. - bga: :ref:`boiler_generator_assn_eia860` table. + bga: :ref:`core_eia860__assn_boiler_generator` table. Returns: table of generators with stacked energy sources and broadcasted net generation - and fuel data from the :ref:`generation_eia923` and :ref:`generation_fuel_eia923` + and fuel data from the :ref:`core_eia923__monthly_generation` and :ref:`core_eia923__monthly_generation_fuel` tables. There are many duplicate values in this output which will later be used in the allocation process in :func:`allocate_gen_fuel_by_gen_esc` and :func:`allocate_fuel_by_gen_esc`. @@ -813,7 +813,7 @@ def remove_inactive_generators(gen_assoc: pd.DataFrame) -> pd.DataFrame: Args: gen_assoc: table of generators with stacked energy sources and broadcasted net - generation data from the generation_eia923 and generation_fuel_eia923 + generation data from the core_eia923__monthly_generation and core_eia923__monthly_generation_fuel tables. Output of :func:`associate_generator_tables`. """ existing = gen_assoc.loc[(gen_assoc.operational_status == "existing")] @@ -979,9 +979,9 @@ def _allocate_unassociated_pm_records( col_w_unexpected_codes: Literal["energy_source_code", "prime_mover_code"], data_columns: list[str], ) -> pd.DataFrame: - """Associate unassociated :ref:`boiler_fuel_eia923` table records on idx_cols. + """Associate unassociated :ref:`core_eia923__monthly_boiler_fuel` table records on idx_cols. - There are a subset of :ref:`boiler_fuel_eia923` and :ref:`generation_fuel_eia923` + There are a subset of :ref:`core_eia923__monthly_boiler_fuel` and :ref:`core_eia923__monthly_generation_fuel` records which do not merge onto the stacked generator table on ``IDX_GENS_PM_ESC`` or ``ID_PM_ESC`` respectively. These records generally don't match with the set of prime movers and energy sources in the stacked generator table. In this method, we @@ -1002,7 +1002,7 @@ def _allocate_unassociated_pm_records( # drop_invalid_rows will fail if there are not unassociated records) if gen_assoc[~connected_mask].empty: logger.info( - "No unassociated boiler_fuel_eia923 or generation_fuel_eia923 records. " + "No unassociated core_eia923__monthly_boiler_fuel or core_eia923__monthly_generation_fuel records. " "Skipping _allocate_unassociated_bf_records" ) return gen_assoc @@ -1103,12 +1103,12 @@ def prep_alloction_fraction(gen_assoc: pd.DataFrame) -> pd.DataFrame: # 'capacity_mw': lambda x: x.sum(min_count=1), # 'in_g_tbl': 'all'},) gen_pm_fuel = ( - gen_assoc.merge( # flag if all generators exist in the generators_eia860 tbl + gen_assoc.merge( # flag if all generators exist in the core_eia860__scd_generators tbl gens_gb_pm_esc[["in_g_tbl"]].all().reset_index(), on=IDX_PM_ESC, suffixes=("", "_all"), ) - .merge( # flag if some generators exist in the generators_eia860 tbl + .merge( # flag if some generators exist in the core_eia860__scd_generators tbl gens_gb_pm_esc[["in_g_tbl", "more_mwh_in_g_than_gf_tbl"]] .any() .reset_index(), @@ -1201,14 +1201,14 @@ def allocate_gen_fuel_by_gen_esc(gen_pm_fuel: pd.DataFrame) -> pd.DataFrame: """Allocate net generation to generators/energy_source_code via three methods. There are three main types of generators: - * "all gen": generators of plants which fully report to the ``generation_eia923`` - table. This includes records that report more MWh to the ``generation_eia923`` - table than to the ``generation_fuel_eia923`` table (if we did not include these + * "all gen": generators of plants which fully report to the ``core_eia923__monthly_generation`` + table. This includes records that report more MWh to the ``core_eia923__monthly_generation`` + table than to the ``core_eia923__monthly_generation_fuel`` table (if we did not include these records, the ). * "some gen": generators of plants which partially report to the - ``generation_eia923`` table. + ``core_eia923__monthly_generation`` table. * "gf only": generators of plants which do not report at all to the - ``generation_eia923`` table. + ``core_eia923__monthly_generation`` table. Each different type of generator needs to be treated slightly differently, but all will end up with a ``frac`` column that can be used to allocate @@ -1324,11 +1324,11 @@ def allocate_fuel_by_gen_esc(gen_pm_fuel: pd.DataFrame) -> pd.DataFrame: There are three main types of generators: * "all bf": generators of plants which fully report to the - boiler_fuel_eia923 table. + core_eia923__monthly_boiler_fuel table. * "some bf": generators of plants which partially report to the - boiler_fuel_eia923 table. + core_eia923__monthly_boiler_fuel table. * "gf only": generators of plants which do not report at all to the - boiler_fuel_eia923 table. + core_eia923__monthly_boiler_fuel table. Each different type of generator needs to be treated slightly differently, but all will end up with a ``frac`` column that can be used to allocate @@ -1498,7 +1498,7 @@ def distribute_annually_reported_data_to_months_if_annual( Note: We should be able to use the ``reporting_frequency_code`` column for the identification of annually reported data. This currently does not work because we assumed this was a plant-level annual attribute (and is thus stored in the - ``plants_eia860`` table). See Issue #1933. + ``core_eia860__scd_plants`` table). See Issue #1933. Args: df: a pandas dataframe, either loaded from pudl_out.gen_original_eia923() or @@ -1785,12 +1785,12 @@ def allocate_bf_data_to_gens( ) -> pd.DataFrame: """Allocates boiler fuel data to the generator level. - Distributes boiler-level data from boiler_fuel_eia923 to the generator level based + Distributes boiler-level data from core_eia923__monthly_boiler_fuel to the generator level based on the boiler-generator association table and the nameplate capacity of the connected generators. - Because fuel consumption in the boiler_fuel_eia923 table is reported per boiler_id, - we must first map this data to generators using the boiler_generator_assn_eia860 + Because fuel consumption in the core_eia923__monthly_boiler_fuel table is reported per boiler_id, + we must first map this data to generators using the core_eia860__assn_boiler_generator table. For boilers that have a 1:m or m: m relationship with generators, we allocate the reported fuel to each associated generator based on the nameplate capacity of each generator. So if boiler "1" was associated with generator A (25 MW) and generator @@ -1977,10 +1977,10 @@ def calc_net_gen_diff(gen_pm_fuel, idx): def test_gen_fuel_allocation( gen: pd.DataFrame, net_gen_alloc: pd.DataFrame, ratio: float = 0.05 ) -> None: - """Does the allocated MWh differ from the granular :ref:`generation_eia923`? + """Does the allocated MWh differ from the granular :ref:`core_eia923__monthly_generation`? Args: - gen: the ``generation_eia923`` table. + gen: the ``core_eia923__monthly_generation`` table. net_gen_alloc: the allocated net generation at the :py:const:`IDX_PM_ESC` level ratio: the tolerance """ @@ -2075,7 +2075,7 @@ def test_original_gf_vs_the_allocated_by_gens_gf( if max_diff > 10 or min_diff < -5: raise AssertionError( f"ahhhHHhh. {data_col} has some plant-year aggregations that that " - "allocated data that is off from the original generation_fuel_eia923 " + "allocated data that is off from the original core_eia923__monthly_generation_fuel " "data by more than an accepted range of tolerance. \n" f" Min difference: {min_diff}\n" f" Max difference: {max_diff}" diff --git a/src/pudl/analysis/classify_plants_ferc1.py b/src/pudl/analysis/classify_plants_ferc1.py index 42535de3ef..227f926747 100644 --- a/src/pudl/analysis/classify_plants_ferc1.py +++ b/src/pudl/analysis/classify_plants_ferc1.py @@ -624,7 +624,7 @@ def fuel_by_plant_ferc1( Args: fuel_df: Pandas DataFrame resembling the post-transform - result for the fuel_ferc1 table. + result for the core_ferc1__yearly_steam_plants_fuel_sched402 table. thresh: A value between 0.5 and 1.0 indicating the minimum fraction of overall heat content that must have been provided by a fuel in a plant-year for it to be considered the "primary" fuel for the plant in that year. @@ -632,11 +632,11 @@ def fuel_by_plant_ferc1( Returns: DataFrame with a single record for each plant-year, including the columns - required to merge it with the plants_steam_ferc1 table/DataFrame (report_year, - utility_id_ferc1, and plant_name) as well as totals for fuel mmbtu consumed in - that plant-year, and the cost of fuel in that year, the proportions of heat - content and fuel costs for each fuel in that year, and a column that labels the - plant's primary fuel for that year. + required to merge it with the :ref:`core_ferc1__yearly_steam_plants_sched402` + table/DataFrame (report_year, utility_id_ferc1, and plant_name) as well as + totals for fuel mmbtu consumed in that plant-year, and the cost of fuel in that + year, the proportions of heat content and fuel costs for each fuel in that year, + and a column that labels the plant's primary fuel for that year. Raises: AssertionError: If the DataFrame input does not have the columns required to @@ -662,7 +662,8 @@ def fuel_by_plant_ferc1( # Calculate per-fuel derived values and add them to the DataFrame df = ( # Really there should *not* be any duplicates here but... there's a - # bug somewhere that introduces them into the fuel_ferc1 table. + # bug somewhere that introduces them into the + # core_ferc1__yearly_steam_plants_fuel_sched402 table. fuel_df[keep_cols] .drop_duplicates() # Calculate totals for each record based on per-unit values: diff --git a/src/pudl/analysis/ferc1_eia_record_linkage.py b/src/pudl/analysis/eia_ferc1_record_linkage.py similarity index 97% rename from src/pudl/analysis/ferc1_eia_record_linkage.py rename to src/pudl/analysis/eia_ferc1_record_linkage.py index 764f08739f..27844f65ec 100644 --- a/src/pudl/analysis/ferc1_eia_record_linkage.py +++ b/src/pudl/analysis/eia_ferc1_record_linkage.py @@ -55,21 +55,23 @@ io_manager_key="pudl_sqlite_io_manager", compute_kind="Python", ) -def out__yearly_plants_all_ferc1_plant_parts_eia( - denorm_plants_all_ferc1: pd.DataFrame, - denorm_fuel_by_plant_ferc1: pd.DataFrame, - plant_parts_eia: pd.DataFrame, +def out_pudl__yearly_assn_eia_ferc1_plant_parts( + out_ferc1__yearly_all_plants: pd.DataFrame, + out_ferc1__yearly_steam_plants_fuel_by_plant_sched402: pd.DataFrame, + out_eia__yearly_plant_parts: pd.DataFrame, ) -> pd.DataFrame: """Coordinate the connection between FERC1 plants and EIA plant-parts. Args: - denorm_plants_all_ferc1: Table of all of the FERC1-reporting plants. - denorm_fuel_by_plant_ferc1: Table of the fuel reported aggregated to the FERC1 - plant-level. - plant_parts_eia: The EIA plant parts list. + out_ferc1__yearly_all_plants: Table of all of the FERC1-reporting plants. + out_ferc1__yearly_steam_plants_fuel_by_plant_sched402: Table of the fuel + reported aggregated to the FERC1 plant-level. + out_eia__yearly_plant_parts: The EIA plant parts list. """ inputs = InputManager( - denorm_plants_all_ferc1, denorm_fuel_by_plant_ferc1, plant_parts_eia + out_ferc1__yearly_all_plants, + out_ferc1__yearly_steam_plants_fuel_by_plant_sched402, + out_eia__yearly_plant_parts, ) # compile/cache inputs upfront. Hopefully we can catch any errors in inputs early. inputs.execute() @@ -96,7 +98,7 @@ def out__yearly_plants_all_ferc1_plant_parts_eia( plants_ferc1=inputs.get_plants_ferc1(), ).pipe(add_null_overrides) # Override specified values with NA record_id_eia connects_ferc1_eia = Resource.from_id( - "out__yearly_plants_all_ferc1_plant_parts_eia" + "out_pudl__yearly_assn_eia_ferc1_plant_parts" ).enforce_schema(connects_ferc1_eia) return connects_ferc1_eia @@ -205,7 +207,9 @@ def get_plants_ferc1(self, clobber: bool = False) -> pd.DataFrame: x.plant_id_report_year + "_" + x.utility_id_pudl.map(str) ), fuel_cost_per_mmbtu=lambda x: (x.fuel_cost / x.fuel_mmbtu), - heat_rate_mmbtu_mwh=lambda x: (x.fuel_mmbtu / x.net_generation_mwh), + unit_heat_rate_mmbtu_per_mwh=lambda x: ( + x.fuel_mmbtu / x.net_generation_mwh + ), ) .rename( columns={ @@ -415,9 +419,9 @@ def make_features( label="fuel_cost_per_mmbtu", ), Numeric( - "heat_rate_mmbtu_mwh", - "heat_rate_mmbtu_mwh", - label="heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", + "unit_heat_rate_mmbtu_per_mwh", + label="unit_heat_rate_mmbtu_per_mwh", ), Exact( "fuel_type_code_pudl", @@ -692,7 +696,7 @@ def prep_train_connections( one_to_many = ( pd.read_csv( importlib.resources.files("pudl.package_data.glue") - / "ferc1_eia_one_to_many.csv" + / "eia_ferc1_one_to_many.csv" ) .pipe(pudl.helpers.cleanstrings_snake, ["record_id_eia"]) .drop_duplicates(subset=["record_id_ferc1", "record_id_eia"]) @@ -740,7 +744,7 @@ def prep_train_connections( train_df = ( pd.read_csv( - importlib.resources.files("pudl.package_data.glue") / "ferc1_eia_train.csv" + importlib.resources.files("pudl.package_data.glue") / "eia_ferc1_train.csv" ) .pipe(pudl.helpers.cleanstrings_snake, ["record_id_eia"]) .drop_duplicates(subset=["record_id_ferc1", "record_id_eia"]) @@ -1028,7 +1032,7 @@ def add_null_overrides(connects_ferc1_eia): logger.info("Overriding specified record_id_ferc1 values with NA record_id_eia") # Get record_id_ferc1 values that should be overriden to have no EIA match null_overrides = pd.read_csv( - importlib.resources.files("pudl.package_data.glue") / "ferc1_eia_null.csv" + importlib.resources.files("pudl.package_data.glue") / "eia_ferc1_null.csv" ).pipe( restrict_train_connections_on_date_range, id_col="record_id_ferc1", @@ -1047,7 +1051,7 @@ def add_null_overrides(connects_ferc1_eia): logger.debug(f"Found {len(null_overrides)} null overrides") # List of EIA columns to null. Ideally would like to get this from elsewhere, but # compiling this here for now... - eia_cols_to_null = Resource.from_id("plant_parts_eia").get_field_names() + eia_cols_to_null = Resource.from_id("out_eia__yearly_plant_parts").get_field_names() # Make all EIA values NA for record_id_ferc1 values in the Null overrides list and # make the match_type column say "overriden" connects_ferc1_eia.loc[ diff --git a/src/pudl/analysis/ferc1_eia_train.py b/src/pudl/analysis/eia_ferc1_train.py similarity index 93% rename from src/pudl/analysis/ferc1_eia_train.py rename to src/pudl/analysis/eia_ferc1_train.py index c66011689e..aa91e04395 100644 --- a/src/pudl/analysis/ferc1_eia_train.py +++ b/src/pudl/analysis/eia_ferc1_train.py @@ -103,7 +103,7 @@ "total_fuel_cost", "total_mmbtu", "fuel_cost_per_mmbtu", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", ] # -------------------------------------------------------------------------------------- @@ -149,25 +149,25 @@ def _is_best_match( return df -def _prep_ferc1_eia(ferc1_eia, utils_eia860) -> pd.DataFrame: +def _prep_eia_ferc1(eia_ferc1, utils_eia860) -> pd.DataFrame: """Prep FERC-EIA for use in override output sheet pre-utility subgroups. Args: - ferc1_eia (pd.DataFrame): The dataframe resulting from pudl_out.ferc1_eia(). + eia_ferc1 (pd.DataFrame): The dataframe resulting from pudl_out.ferc1_eia(). utils_eia860 (pd.DataFrame): The dataframe resulting from pudl_out.utils_eia860. Returns: - pd.DataFrame: A version of the ferc1_eia table that's been modified for + pd.DataFrame: A version of the eia_ferc1 table that's been modified for the purposes of creating an manual mapping spreadsheet. """ logger.debug("Prepping FERC-EIA table") # Only want to keep the plant_name_ppe field which replaces plant_name_eia - ferc1_eia_prep = ferc1_eia.copy().drop(columns="plant_name_eia") + eia_ferc1_prep = eia_ferc1.copy().drop(columns="plant_name_eia") # Add utility_name_eia - this must happen before renaming the cols or else there # will be duplicate utility_name_eia columns. utils_eia860.loc[:, "report_year"] = utils_eia860.report_date.dt.year - ferc1_eia_prep = pd.merge( - ferc1_eia_prep, + eia_ferc1_prep = pd.merge( + eia_ferc1_prep, utils_eia860[["utility_id_eia", "utility_name_eia", "report_year"]].copy(), on=["utility_id_eia", "report_year"], how="left", @@ -176,41 +176,41 @@ def _prep_ferc1_eia(ferc1_eia, utils_eia860) -> pd.DataFrame: # Add the new columns to the df for new_col in [x for x in RENAME_COLS_FERC1_EIA if "new_" in x]: - ferc1_eia_prep.loc[:, new_col] = pd.NA + eia_ferc1_prep.loc[:, new_col] = pd.NA # Rename the columns, and remove unwanted columns from ferc-eia table - ferc1_eia_prep = ferc1_eia_prep.rename(columns=RENAME_COLS_FERC1_EIA)[ + eia_ferc1_prep = eia_ferc1_prep.rename(columns=RENAME_COLS_FERC1_EIA)[ list(RENAME_COLS_FERC1_EIA.values()) ] # Add in pct diff values for pct_diff_col in [x for x in RENAME_COLS_FERC1_EIA.values() if "_pct_diff" in x]: - ferc1_eia_prep = _pct_diff(ferc1_eia_prep, pct_diff_col) + eia_ferc1_prep = _pct_diff(eia_ferc1_prep, pct_diff_col) # Add in fuel_type_code_pudl diff (qualitative bool) - ferc1_eia_prep["fuel_type_code_pudl_diff"] = False - ferc1_eia_prep_nona = ferc1_eia_prep[ - ferc1_eia_prep.fuel_type_code_pudl_eia.notna() - & ferc1_eia_prep.fuel_type_code_pudl_ferc1.notna() + eia_ferc1_prep["fuel_type_code_pudl_diff"] = False + eia_ferc1_prep_nona = eia_ferc1_prep[ + eia_ferc1_prep.fuel_type_code_pudl_eia.notna() + & eia_ferc1_prep.fuel_type_code_pudl_ferc1.notna() ].copy() - ferc1_eia_prep_nona["fuel_type_code_pudl_diff"] = ( - ferc1_eia_prep_nona.fuel_type_code_pudl_eia - == ferc1_eia_prep_nona.fuel_type_code_pudl_ferc1 + eia_ferc1_prep_nona["fuel_type_code_pudl_diff"] = ( + eia_ferc1_prep_nona.fuel_type_code_pudl_eia + == eia_ferc1_prep_nona.fuel_type_code_pudl_ferc1 ) - ferc1_eia_prep.update(ferc1_eia_prep_nona) + eia_ferc1_prep.update(eia_ferc1_prep_nona) # Add in installation_year diff (diff vs. pct_diff) - ferc1_eia_prep.loc[ - ferc1_eia_prep.installation_year_ferc1.notna(), "installation_year_ferc1" - ] = ferc1_eia_prep.installation_year_ferc1.astype("Int64") + eia_ferc1_prep.loc[ + eia_ferc1_prep.installation_year_ferc1.notna(), "installation_year_ferc1" + ] = eia_ferc1_prep.installation_year_ferc1.astype("Int64") - ferc1_eia_prep.loc[ - ferc1_eia_prep.installation_year_eia.notna() - & ferc1_eia_prep.installation_year_ferc1.notna(), + eia_ferc1_prep.loc[ + eia_ferc1_prep.installation_year_eia.notna() + & eia_ferc1_prep.installation_year_ferc1.notna(), "installation_year_diff", - ] = ferc1_eia_prep.installation_year_eia - ferc1_eia_prep.installation_year_ferc1 + ] = eia_ferc1_prep.installation_year_eia - eia_ferc1_prep.installation_year_ferc1 # Add best match col - ferc1_eia_prep = _is_best_match(ferc1_eia_prep) - return ferc1_eia_prep + eia_ferc1_prep = _is_best_match(eia_ferc1_prep) + return eia_ferc1_prep def _prep_ppe(ppe, utils_eia860) -> pd.DataFrame: @@ -289,9 +289,9 @@ def _get_util_year_subsets(inputs_dict, util_id_eia_list, years) -> dict: or year subset" ) - if df_name == "ferc1_eia": + if df_name == "eia_ferc1": # Add column with excel formula to check if the override record id is the - # same as the AI assigend id. Doing this here instead of prep_ferc1_eia + # same as the AI assigend id. Doing this here instead of prep_eia_ferc1 # because it is based on row index number which is changes when you take a # subset of the data. subset_df = subset_df.reset_index(drop=True) @@ -336,7 +336,7 @@ def _output_override_spreadsheet( def generate_all_override_spreadsheets( - ferc1_eia, ppe, utils_eia860, util_dict, years, output_dir_path + eia_ferc1, ppe, utils_eia860, util_dict, years, output_dir_path ) -> None: """Output override spreadsheets for all specified utilities and years. @@ -344,7 +344,7 @@ def generate_all_override_spreadsheets( output directory. Args: - ferc1_eia (pd.DataFrame): The dataframe resulting from pudl_out.ferc1_eia(). + eia_ferc1 (pd.DataFrame): The dataframe resulting from pudl_out.ferc1_eia(). ppe (pd.DataFrame): The dataframe resulting from pudl_out.plant_parts_eia utils_eia860 (pd.DataFrame): The dataframe resulting from pudl_out.utils_eia860. util_dict (dict): A dictionary with keys that are the names of utility @@ -358,7 +358,7 @@ def generate_all_override_spreadsheets( # Generate full input tables # inputs_dict = _generate_input_dfs(pudl_out) inputs_dict = { - "ferc1_eia": _prep_ferc1_eia(ferc1_eia, utils_eia860), + "eia_ferc1": _prep_eia_ferc1(eia_ferc1, utils_eia860), "ppe": _prep_ppe(ppe, utils_eia860), } @@ -425,7 +425,7 @@ def check_if_already_in_training(training_data, validated_connections): def validate_override_fixes( validated_connections, ppe, - ferc1_eia, + eia_ferc1, training_data, expect_override_overrides=False, allow_mismatched_utilities=True, @@ -437,7 +437,7 @@ def validate_override_fixes( directory that is ready to be added to be validated and subsumed into the training data. ppe (pd.DataFrame): The dataframe resulting from pudl_out.plant_parts_eia - ferc1_eia (pd.DataFrame): The dataframe resulting from pudl_out.ferc1_eia + eia_ferc1 (pd.DataFrame): The dataframe resulting from pudl_out.ferc1_eia training_data (pd.DataFrame): The current FERC-EIA training data expect_override_overrides (boolean): Whether you expect the tables to have overridden matches already in the training data. @@ -494,7 +494,7 @@ def validate_override_fixes( ) # It's unlikely that this changed, but check FERC id too just in case! - actual_ferc_ids = ferc1_eia.record_id_ferc1.unique() + actual_ferc_ids = eia_ferc1.record_id_ferc1.unique() _check_id_consistency( "record_id_ferc1", only_overrides, actual_ferc_ids, "values that don't exist" ) @@ -684,7 +684,7 @@ def _add_to_one_to_many_overrides(one_to_many, current_one_to_many_path) -> None def validate_and_add_to_training( utils_eia860, ppe, - ferc1_eia, + eia_ferc1, input_dir_path, expect_override_overrides=False, allow_mismatched_utilities=True, @@ -710,12 +710,12 @@ def validate_and_add_to_training( pandas.DataFrame: A DataFrame with all of the new overrides combined. """ glue_resource_path = importlib.resources.files("pudl.package_data.glue") - path_to_current_training = glue_resource_path / "ferc1_eia_train.csv" + path_to_current_training = glue_resource_path / "eia_ferc1_train.csv" path_to_new_training = input_dir_path current_training_df = pd.read_csv(path_to_current_training) - path_to_null_overrides = glue_resource_path / "ferc1_eia_null.csv" + path_to_null_overrides = glue_resource_path / "eia_ferc1_null.csv" if one_to_many: - path_to_one_to_many = glue_resource_path / "ferc1_eia_one_to_many.csv" + path_to_one_to_many = glue_resource_path / "eia_ferc1_one_to_many.csv" override_cols = [ "record_id_eia", "record_id_ferc1", @@ -740,7 +740,7 @@ def validate_and_add_to_training( file_df = file_raw.pipe( validate_override_fixes, ppe, - ferc1_eia, + eia_ferc1, current_training_df, expect_override_overrides=expect_override_overrides, allow_mismatched_utilities=allow_mismatched_utilities, @@ -764,7 +764,7 @@ def validate_and_add_to_training( multi_file_df = multi_df.pipe( validate_override_fixes, ppe=ppe, - ferc1_eia=ferc1_eia, + eia_ferc1=eia_ferc1, training_data=current_training_df, expect_override_overrides=expect_override_overrides, allow_mismatched_utilities=allow_mismatched_utilities, diff --git a/src/pudl/analysis/epacamd_eia.py b/src/pudl/analysis/epacamd_eia.py index 5d093c9b29..d829d3eb34 100644 --- a/src/pudl/analysis/epacamd_eia.py +++ b/src/pudl/analysis/epacamd_eia.py @@ -7,8 +7,8 @@ Usage Example: epacems = pudl.output.epacems.epacems(states=['ID'], years=[2020]) # subset for test -epacamd_eia = pudl_out.epacamd_eia() -filtered_crosswalk = filter_crosswalk(epacamd_eia, epacems) +core_epa__assn_eia_epacamd = pudl_out.epacamd_eia() +filtered_crosswalk = filter_crosswalk(core_epa__assn_eia_epacamd, epacems) crosswalk_with_subplant_ids = pudl.etl.make_subplant_ids(filtered_crosswalk) """ @@ -37,18 +37,18 @@ def _get_unique_keys(epacems: pd.DataFrame | dd.DataFrame) -> pd.DataFrame: def filter_crosswalk_by_epacems( crosswalk: pd.DataFrame, epacems: pd.DataFrame | dd.DataFrame ) -> pd.DataFrame: - """Inner join unique CEMS units with the epacamd_eia crosswalk. + """Inner join unique CEMS units with the core_epa__assn_eia_epacamd crosswalk. This is essentially an empirical filter on EPA units. Instead of filtering by construction/retirement dates in the crosswalk (thus assuming they are accurate), use the presence/absence of CEMS data to filter the units. Args: - crosswalk: epacamd_eia crosswalk + crosswalk: core_epa__assn_eia_epacamd crosswalk unique_epacems_ids (pd.DataFrame): unique ids from _get_unique_keys Returns: - The inner join of the epacamd_eia crosswalk and unique epacems units. Adds + The inner join of the core_epa__assn_eia_epacamd crosswalk and unique epacems units. Adds the global ID column unit_id_epa. """ unique_epacems_ids = _get_unique_keys(epacems) @@ -64,10 +64,10 @@ def filter_out_boiler_rows(crosswalk: pd.DataFrame) -> pd.DataFrame: """Remove rows that represent graph edges between generators and boilers. Args: - crosswalk (pd.DataFrame): epacamd_eia crosswalk + crosswalk (pd.DataFrame): core_epa__assn_eia_epacamd crosswalk Returns: - pd.DataFrame: the epacamd_eia crosswalk with boiler rows (many/one-to-many) + pd.DataFrame: the core_epa__assn_eia_epacamd crosswalk with boiler rows (many/one-to-many) removed """ crosswalk = crosswalk.drop_duplicates( @@ -82,12 +82,12 @@ def filter_crosswalk( """Remove unmapped crosswalk rows or duplicates due to m2m boiler relationships. Args: - crosswalk (pd.DataFrame): The epacamd_eia crosswalk. + crosswalk (pd.DataFrame): The core_epa__assn_eia_epacamd crosswalk. epacems (Union[pd.DataFrame, dd.DataFrame]): Emissions data. Must contain columns named ["plant_id_eia", "emissions_unit_id_epa"] Returns: - pd.DataFrame: A filtered copy of epacamd_eia crosswalk + pd.DataFrame: A filtered copy of core_epa__assn_eia_epacamd crosswalk """ filtered_crosswalk = filter_out_boiler_rows(crosswalk) key_map = filter_crosswalk_by_epacems(filtered_crosswalk, epacems) diff --git a/src/pudl/analysis/mcoe.py b/src/pudl/analysis/mcoe.py index e02e580c62..e5b3f957b1 100644 --- a/src/pudl/analysis/mcoe.py +++ b/src/pudl/analysis/mcoe.py @@ -41,7 +41,6 @@ def mcoe_asset_factory( freq: Literal["AS", "MS"], - io_manager_key: str | None = None, ) -> list[AssetsDefinition]: """Build MCOE related assets at yearly and monthly frequencies.""" agg_freqs = {"AS": "yearly", "MS": "monthly"} @@ -49,28 +48,28 @@ def mcoe_asset_factory( raise ValueError(f"freq must be one of {agg_freqs.keys()}, got: {freq}.") @asset( - name=f"heat_rate_by_unit_{agg_freqs[freq]}", + name=f"_out_eia__{agg_freqs[freq]}_heat_rate_by_unit", ins={ "gen": AssetIn( - key=f"generation_fuel_by_generator_energy_source_{agg_freqs[freq]}_eia923" + key=f"out_eia923__{agg_freqs[freq]}_generation_fuel_by_generator_energy_source" ), - "bga": AssetIn(key="boiler_generator_assn_eia860"), + "bga": AssetIn(key="core_eia860__assn_boiler_generator"), }, - io_manager_key=io_manager_key, compute_kind="Python", + io_manager_key="pudl_sqlite_io_manager", ) def hr_by_unit_asset(gen: pd.DataFrame, bga: pd.DataFrame) -> pd.DataFrame: return heat_rate_by_unit(gen_fuel_by_energy_source=gen, bga=bga) @asset( - name=f"heat_rate_by_generator_{agg_freqs[freq]}", + name=f"_out_eia__{agg_freqs[freq]}_heat_rate_by_generator", ins={ - "bga": AssetIn(key="boiler_generator_assn_eia860"), - "hr_by_unit": AssetIn(key=f"heat_rate_by_unit_{agg_freqs[freq]}"), - "gens": AssetIn(key="denorm_generators_eia"), + "bga": AssetIn(key="core_eia860__assn_boiler_generator"), + "hr_by_unit": AssetIn(key=f"_out_eia__{agg_freqs[freq]}_heat_rate_by_unit"), + "gens": AssetIn(key="_out_eia__yearly_generators"), }, - io_manager_key=io_manager_key, compute_kind="Python", + io_manager_key="pudl_sqlite_io_manager", ) def hr_by_gen_asset( bga: pd.DataFrame, hr_by_unit: pd.DataFrame, gens: pd.DataFrame @@ -78,14 +77,16 @@ def hr_by_gen_asset( return heat_rate_by_gen(bga=bga, hr_by_unit=hr_by_unit, gens=gens) @asset( - name=f"fuel_cost_by_generator_{agg_freqs[freq]}", + name=f"_out_eia__{agg_freqs[freq]}_fuel_cost_by_generator", ins={ - "hr_by_gen": AssetIn(key=f"heat_rate_by_generator_{agg_freqs[freq]}"), - "gens": AssetIn(key="denorm_generators_eia"), - "frc": AssetIn(key=f"denorm_fuel_receipts_costs_{agg_freqs[freq]}_eia923"), + "hr_by_gen": AssetIn( + key=f"_out_eia__{agg_freqs[freq]}_heat_rate_by_generator" + ), + "gens": AssetIn(key="_out_eia__yearly_generators"), + "frc": AssetIn(key=f"out_eia923__{agg_freqs[freq]}_fuel_receipts_costs"), }, - io_manager_key=io_manager_key, compute_kind="Python", + io_manager_key="pudl_sqlite_io_manager", ) def fc_asset( hr_by_gen: pd.DataFrame, gens: pd.DataFrame, frc: pd.DataFrame @@ -93,28 +94,29 @@ def fc_asset( return fuel_cost(hr_by_gen=hr_by_gen, gens=gens, frc=frc) @asset( - name=f"capacity_factor_by_generator_{agg_freqs[freq]}", + name=f"_out_eia__{agg_freqs[freq]}_capacity_factor_by_generator", ins={ "gen": AssetIn( - key=f"generation_fuel_by_generator_{agg_freqs[freq]}_eia923" + key=f"out_eia923__{agg_freqs[freq]}_generation_fuel_by_generator" ), - "gens": AssetIn(key="denorm_generators_eia"), + "gens": AssetIn(key="_out_eia__yearly_generators"), }, - io_manager_key=io_manager_key, compute_kind="Python", + io_manager_key="pudl_sqlite_io_manager", ) def cf_asset(gens: pd.DataFrame, gen: pd.DataFrame) -> pd.DataFrame: return capacity_factor(gens=gens, gen=gen, freq=freq) @asset( - name=f"mcoe_{agg_freqs[freq]}", + name=f"_out_eia__{agg_freqs[freq]}_derived_generator_attributes", ins={ - "fuel_cost": AssetIn(key=f"fuel_cost_by_generator_{agg_freqs[freq]}"), + "fuel_cost": AssetIn( + key=f"_out_eia__{agg_freqs[freq]}_fuel_cost_by_generator" + ), "capacity_factor": AssetIn( - key=f"capacity_factor_by_generator_{agg_freqs[freq]}" + key=f"_out_eia__{agg_freqs[freq]}_capacity_factor_by_generator" ), }, - io_manager_key=io_manager_key, compute_kind="Python", config_schema={ "min_heat_rate": Field( @@ -155,6 +157,7 @@ def cf_asset(gens: pd.DataFrame, gen: pd.DataFrame) -> pd.DataFrame: ), ), }, + io_manager_key="pudl_sqlite_io_manager", ) def mcoe_asset( context, fuel_cost: pd.DataFrame, capacity_factor: pd.DataFrame @@ -169,12 +172,14 @@ def mcoe_asset( ) @asset( - name=f"mcoe_generators_{agg_freqs[freq]}", + name=f"out_eia__{agg_freqs[freq]}_generators", ins={ - "mcoe": AssetIn(key=f"mcoe_{agg_freqs[freq]}"), - "gens": AssetIn(key="denorm_generators_eia"), + "mcoe": AssetIn( + key=f"_out_eia__{agg_freqs[freq]}_derived_generator_attributes" + ), + "gens": AssetIn(key="_out_eia__yearly_generators"), }, - io_manager_key=io_manager_key, + io_manager_key="pudl_sqlite_io_manager", compute_kind="Python", config_schema={ "all_gens": Field( @@ -182,7 +187,7 @@ def mcoe_asset( default_value=True, description=( "If True, include attributes of all generators in the " - ":ref:`generators_eia860` table, rather than just the generators " + ":ref:`core_eia860__scd_generators` table, rather than just the generators " "which have records in the derived MCOE values. True by default." ), ), @@ -223,7 +228,6 @@ def mcoe_generators_asset( for freq in ["AS", "MS"] for mcoe_asset in mcoe_asset_factory( freq=freq, - io_manager_key="pudl_sqlite_io_manager", ) ] @@ -255,7 +259,7 @@ def heat_rate_by_unit(gen_fuel_by_energy_source: pd.DataFrame, bga: pd.DataFrame - unit_id_pudl - net_generation_mwh - fuel_consumed_for_electricity_mmbtu - - heat_rate_mmbtu_mwh + - unit_heat_rate_mmbtu_per_mwh """ gen_fuel_by_unit = pudl.helpers.date_merge( left=gen_fuel_by_energy_source, @@ -271,7 +275,7 @@ def heat_rate_by_unit(gen_fuel_by_energy_source: pd.DataFrame, bga: pd.DataFrame .sum() .convert_dtypes() .assign( - heat_rate_mmbtu_mwh=lambda x: x.fuel_consumed_for_electricity_mmbtu + unit_heat_rate_mmbtu_per_mwh=lambda x: x.fuel_consumed_for_electricity_mmbtu / x.net_generation_mwh ) ) @@ -296,7 +300,7 @@ def heat_rate_by_gen( Returns: DataFrame with columns report_date, plant_id_eia, unit_id_pudl, generator_id, - heat_rate_mmbtu_mwh, fuel_type_code_pudl, fuel_type_count, prime_mover_code. + unit_heat_rate_mmbtu_per_mwh, fuel_type_code_pudl, fuel_type_count, prime_mover_code. The output will have a time frequency corresponding to that of the input pudl_out. Output data types are set to their canonical values before returning. """ @@ -309,7 +313,7 @@ def heat_rate_by_gen( "report_date", "plant_id_eia", "unit_id_pudl", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", ], ] @@ -374,7 +378,7 @@ def fuel_cost( "generator_id", "unit_id_pudl", "report_date", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", ], ] gens = gens.loc[ @@ -485,7 +489,7 @@ def fuel_cost( "plant_id_eia", "report_date", "generator_id", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "fuel_cost_from_eiaapi", ] ], @@ -502,7 +506,7 @@ def fuel_cost( "report_date", "generator_id", "fuel_cost_per_mmbtu", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "fuel_cost_from_eiaapi", ] ] @@ -510,13 +514,14 @@ def fuel_cost( fc = ( pd.concat([one_fuel, multi_fuel], sort=True) .assign( - fuel_cost_per_mwh=lambda x: x.fuel_cost_per_mmbtu * x.heat_rate_mmbtu_mwh + fuel_cost_per_mwh=lambda x: x.fuel_cost_per_mmbtu + * x.unit_heat_rate_mmbtu_per_mwh ) .sort_values(["report_date", "plant_id_eia", "generator_id"]) ) out_df = ( - gen_w_ft.drop("heat_rate_mmbtu_mwh", axis=1) + gen_w_ft.drop("unit_heat_rate_mmbtu_per_mwh", axis=1) .drop_duplicates() .merge(fc, on=["report_date", "plant_id_eia", "generator_id"]) ) @@ -603,7 +608,7 @@ def mcoe( "unit_id_pudl", "fuel_cost_from_eiaapi", "fuel_cost_per_mmbtu", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "fuel_cost_per_mwh", ], ], @@ -615,12 +620,12 @@ def mcoe( ) # Calculate a couple more derived values: .assign( - total_mmbtu=lambda x: x.net_generation_mwh * x.heat_rate_mmbtu_mwh, + total_mmbtu=lambda x: x.net_generation_mwh * x.unit_heat_rate_mmbtu_per_mwh, total_fuel_cost=lambda x: x.total_mmbtu * x.fuel_cost_per_mmbtu, ) .pipe( pudl.helpers.oob_to_nan_with_dependent_cols, - cols=["heat_rate_mmbtu_mwh"], + cols=["unit_heat_rate_mmbtu_per_mwh"], dependent_cols=["total_mmbtu", "fuel_cost_per_mwh"], lb=min_heat_rate, ub=None, @@ -667,7 +672,7 @@ def mcoe_generators( mcoe: The MCOE dataframe outputted from the `mcoe` analysis function. gens: The denormalized dataframe of all EIA generators. all_gens: if True, include attributes of all generators in the - :ref:`generators_eia860` table, rather than just the generators + :ref:`core_eia860__scd_generators` table, rather than just the generators which have records in the derived MCOE values. True by default. timeseries_fillin: if True, fill in the full timeseries for each generator in the output dataframe. The data in the timeseries will be filled diff --git a/src/pudl/analysis/plant_parts_eia.py b/src/pudl/analysis/plant_parts_eia.py index 16e14ed2ad..d952e16da7 100644 --- a/src/pudl/analysis/plant_parts_eia.py +++ b/src/pudl/analysis/plant_parts_eia.py @@ -281,7 +281,7 @@ WTAVG_DICT = { "fuel_cost_per_mwh": "capacity_mw", - "heat_rate_mmbtu_mwh": "capacity_mw", + "unit_heat_rate_mmbtu_per_mwh": "capacity_mw", "fuel_cost_per_mmbtu": "capacity_mw", } """Dict: a dictionary of columns (keys) to perform weighted averages on and the weight @@ -367,34 +367,32 @@ @asset( - name="mega_generators_eia", compute_kind="Python", ) -def mega_gens_asset( - mcoe_generators_yearly: pd.DataFrame, denorm_ownership_eia860: pd.DataFrame +def out_eia__yearly_generators_by_ownership( + out_eia__yearly_generators: pd.DataFrame, out_eia860__yearly_ownership: pd.DataFrame ) -> pd.DataFrame: """Create mega generators table asset.""" return MakeMegaGenTbl().execute( - mcoe=mcoe_generators_yearly, - own_eia860=denorm_ownership_eia860, + mcoe=out_eia__yearly_generators, + own_eia860=out_eia860__yearly_ownership, ) @asset( - name="plant_parts_eia", io_manager_key="pudl_sqlite_io_manager", compute_kind="Python", ) -def plant_parts_eia_asset( - mega_generators_eia: pd.DataFrame, - denorm_plants_eia: pd.DataFrame, - denorm_utilities_eia: pd.DataFrame, +def out_eia__yearly_plant_parts( + out_eia__yearly_generators_by_ownership: pd.DataFrame, + out_eia__yearly_plants: pd.DataFrame, + out_eia__yearly_utilities: pd.DataFrame, ) -> pd.DataFrame: """Create plant parts list asset.""" return MakePlantParts().execute( - gens_mega=mega_generators_eia, - plants_eia860=denorm_plants_eia, - utils_eia860=denorm_utilities_eia, + gens_mega=out_eia__yearly_generators_by_ownership, + plants_eia860=out_eia__yearly_plants, + utils_eia860=out_eia__yearly_utilities, ) @@ -647,7 +645,7 @@ def execute(self, gens_mega, plants_eia860, utils_eia860): plant_parts_eia=plant_parts_eia, part_name="plant_match_ferc1", path_to_one_to_many=resources.files("pudl.package_data.glue").joinpath( - "ferc1_eia_one_to_many.csv", + "eia_ferc1_one_to_many.csv", ), ) self.plant_parts_eia = TrueGranLabeler().execute(self.plant_parts_eia) @@ -660,7 +658,7 @@ def execute(self, gens_mega, plants_eia860, utils_eia860): ) .pipe(pudl.helpers.organize_cols, FIRST_COLS) .pipe(self._clean_plant_parts) - .pipe(Resource.from_id("plant_parts_eia").format_df) + .pipe(Resource.from_id("out_eia__yearly_plant_parts").format_df) ) return self.plant_parts_eia @@ -761,7 +759,7 @@ def add_one_to_many( ) ] - assert double_df.empty, f"The following record ids have >1 faked part. Double-check these records or move them to the ferc1_eia_null.csv: {one_to_many.loc[one_to_many.gen_id.isin(orig_ids.record_id_eia), 'record_id_ferc1'].drop_duplicates().tolist()}" + assert double_df.empty, f"The following record ids have >1 faked part. Double-check these records or move them to the eia_ferc1_null.csv: {one_to_many.loc[one_to_many.gen_id.isin(orig_ids.record_id_eia), 'record_id_ferc1'].drop_duplicates().tolist()}" return pd.concat([plant_parts_eia, part_df]) diff --git a/src/pudl/analysis/service_territory.py b/src/pudl/analysis/service_territory.py index 6e5c50f3a1..70e0ba54ef 100644 --- a/src/pudl/analysis/service_territory.py +++ b/src/pudl/analysis/service_territory.py @@ -29,11 +29,12 @@ MAP_CRS = "EPSG:3857" # For mapping w/ OSM baselayer tiles CALC_CRS = "ESRI:102003" # For accurate area calculations -ENTITY_TYPE = {"ba": "balancing_authority", "util": "utility"} +ENTITY_TYPE = {"ba": "balancing_authorities", "util": "utilities"} def utility_ids_all_eia( - denorm_utilities_eia: pd.DataFrame, service_territory_eia861: pd.DataFrame + out_eia__yearly_utilities: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, ) -> pd.DataFrame: """Compile IDs and Names of all known EIA Utilities. @@ -43,8 +44,8 @@ def utility_ids_all_eia( process and PUDL database yet. Args: - denorm_utilities_eia: De-normalized EIA 860 utility attributes table. - service_territory_eia861: Normalized EIA 861 Service Territory table. + out_eia__yearly_utilities: De-normalized EIA 860 utility attributes table. + core_eia861__yearly_service_territory: Normalized EIA 861 Service Territory table. Returns: A DataFrame having 2 columns ``utility_id_eia`` and ``utility_name_eia``. @@ -52,8 +53,10 @@ def utility_ids_all_eia( return ( pd.concat( [ - denorm_utilities_eia[["utility_id_eia", "utility_name_eia"]], - service_territory_eia861[["utility_id_eia", "utility_name_eia"]], + out_eia__yearly_utilities[["utility_id_eia", "utility_name_eia"]], + core_eia861__yearly_service_territory[ + ["utility_id_eia", "utility_name_eia"] + ], ] ) .dropna(subset=["utility_id_eia"]) @@ -68,7 +71,7 @@ def get_territory_fips( ids: Iterable[int], assn: pd.DataFrame, assn_col: str, - service_territory_eia861: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, limit_by_state: bool = True, ) -> pd.DataFrame: """Compile county FIPS codes associated with an entity's service territory. @@ -86,7 +89,7 @@ def get_territory_fips( assn_col: Label of the dataframe column in ``assn`` that contains the ID of the entities of interest. Should probably be either ``balancing_authority_id_eia`` or ``utility_id_eia``. - service_territory_eia861: The EIA 861 Service Territory table. + core_eia861__yearly_service_territory: The EIA 861 Service Territory table. limit_by_state: Whether to require that the counties associated with the balancing authority are inside a state that has also been seen in association with the balancing authority and the utility @@ -104,7 +107,7 @@ def get_territory_fips( assn = assn.drop("state", axis="columns") return ( - pd.merge(assn, service_territory_eia861, how="inner") + pd.merge(assn, core_eia861__yearly_service_territory, how="inner") .loc[ :, [ @@ -197,7 +200,7 @@ def get_territory_geometries( ids: Iterable[int], assn: pd.DataFrame, assn_col: str, - service_territory_eia861: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, census_gdf: gpd.GeoDataFrame, limit_by_state: bool = True, dissolve: bool = False, @@ -225,7 +228,7 @@ def get_territory_geometries( assn_col: Label of the dataframe column in ``assn`` that contains the ID of the entities of interest. Should probably be either ``balancing_authority_id_eia`` or ``utility_id_eia``. - service_territory_eia861: The EIA 861 Service Territory table. + core_eia861__yearly_service_territory: The EIA 861 Service Territory table. census_gdf: The US Census DP1 county-level geometries. limit_by_state: Whether to require that the counties associated with the balancing authority are inside a state that has also been @@ -246,7 +249,7 @@ def get_territory_geometries( ids=ids, assn=assn, assn_col=assn_col, - service_territory_eia861=service_territory_eia861, + core_eia861__yearly_service_territory=core_eia861__yearly_service_territory, limit_by_state=limit_by_state, ).pipe( add_geometries, @@ -294,11 +297,11 @@ def _save_geoparquet( def compile_geoms( - balancing_authority_eia861: pd.DataFrame, - balancing_authority_assn_eia861: pd.DataFrame, - denorm_utilities_eia: pd.DataFrame, - service_territory_eia861: pd.DataFrame, - utility_assn_eia861: pd.DataFrame, + core_eia861__yearly_balancing_authority: pd.DataFrame, + core_eia861__assn_balancing_authority: pd.DataFrame, + out_eia__yearly_utilities: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, + core_eia861__assn_utility: pd.DataFrame, census_counties: pd.DataFrame, entity_type: Literal["ba", "util"], save_format: Literal["geoparquet", "geodataframe", "dataframe"], @@ -326,23 +329,31 @@ def compile_geoms( def _limit_years(df: pd.DataFrame) -> pd.DataFrame: return df[df.report_date.dt.year.isin(years)] - balancing_authority_eia861 = _limit_years(balancing_authority_eia861) - balancing_authority_assn_eia861 = _limit_years(balancing_authority_assn_eia861) - denorm_utilities_eia = _limit_years(denorm_utilities_eia) - service_territory_eia861 = _limit_years(service_territory_eia861) - utility_assn_eia861 = _limit_years(utility_assn_eia861) + core_eia861__yearly_balancing_authority = _limit_years( + core_eia861__yearly_balancing_authority + ) + core_eia861__assn_balancing_authority = _limit_years( + core_eia861__assn_balancing_authority + ) + out_eia__yearly_utilities = _limit_years(out_eia__yearly_utilities) + core_eia861__yearly_service_territory = _limit_years( + core_eia861__yearly_service_territory + ) + core_eia861__assn_utility = _limit_years(core_eia861__assn_utility) utilids_all_eia = utility_ids_all_eia( - denorm_utilities_eia, service_territory_eia861 + out_eia__yearly_utilities, core_eia861__yearly_service_territory ) if entity_type == "ba": - ids = balancing_authority_eia861.balancing_authority_id_eia.unique() - assn = balancing_authority_assn_eia861 + ids = ( + core_eia861__yearly_balancing_authority.balancing_authority_id_eia.unique() + ) + assn = core_eia861__assn_balancing_authority assn_col = "balancing_authority_id_eia" elif entity_type == "util": ids = utilids_all_eia.utility_id_eia.unique() - assn = utility_assn_eia861 + assn = core_eia861__assn_utility assn_col = "utility_id_eia" else: raise ValueError(f"Got {entity_type=}, but need either 'ba' or 'util'") @@ -352,7 +363,7 @@ def _limit_years(df: pd.DataFrame) -> pd.DataFrame: ids=ids, assn=assn, assn_col=assn_col, - service_territory_eia861=service_territory_eia861, + core_eia861__yearly_service_territory=core_eia861__yearly_service_territory, census_gdf=census_counties, limit_by_state=limit_by_state, dissolve=dissolve, @@ -379,7 +390,7 @@ def compiled_geoms_asset_factory( """Build asset definitions for balancing authority and utility geometries.""" @asset( - name=f"compiled_geometry_{ENTITY_TYPE[entity_type]}_eia861", + name=f"out_eia861__compiled_geometry_{ENTITY_TYPE[entity_type]}", io_manager_key=io_manager_key, config_schema={ "dissolve": Field( @@ -408,12 +419,12 @@ def compiled_geoms_asset_factory( ) def dagster_compile_geoms( context, - balancing_authority_eia861: pd.DataFrame, - balancing_authority_assn_eia861: pd.DataFrame, - denorm_utilities_eia: pd.DataFrame, - service_territory_eia861: pd.DataFrame, - utility_assn_eia861: pd.DataFrame, - county_censusdp1: pd.DataFrame, + core_eia861__yearly_balancing_authority: pd.DataFrame, + core_eia861__assn_balancing_authority: pd.DataFrame, + out_eia__yearly_utilities: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, + core_eia861__assn_utility: pd.DataFrame, + core_censusdp1__entity_county: pd.DataFrame, ): """Compile all available utility or balancing authority geometries. @@ -426,12 +437,12 @@ def dagster_compile_geoms( save_format = context.op_config["save_format"] return compile_geoms( - balancing_authority_eia861=balancing_authority_eia861, - balancing_authority_assn_eia861=balancing_authority_assn_eia861, - denorm_utilities_eia=denorm_utilities_eia, - service_territory_eia861=service_territory_eia861, - utility_assn_eia861=utility_assn_eia861, - census_counties=county_censusdp1, + core_eia861__yearly_balancing_authority=core_eia861__yearly_balancing_authority, + core_eia861__assn_balancing_authority=core_eia861__assn_balancing_authority, + out_eia__yearly_utilities=out_eia__yearly_utilities, + core_eia861__yearly_service_territory=core_eia861__yearly_service_territory, + core_eia861__assn_utility=core_eia861__assn_utility, + census_counties=core_censusdp1__entity_county, entity_type=entity_type, dissolve=dissolve, limit_by_state=limit_by_state, @@ -699,17 +710,19 @@ def pudl_service_territories( ) _ = compile_geoms( - balancing_authority_eia861=pd.read_sql( - "balancing_authority_eia861", + core_eia861__yearly_balancing_authority=pd.read_sql( + "core_eia861__yearly_balancing_authority", pudl_engine, ), - balancing_authority_assn_eia861=pd.read_sql( - "balancing_authority_assn_eia861", + core_eia861__assn_balancing_authority=pd.read_sql( + "core_eia861__assn_balancing_authority", pudl_engine, ), - denorm_utilities_eia=pd.read_sql("denorm_utilities_eia", pudl_engine), - service_territory_eia861=pd.read_sql("service_territory_eia861", pudl_engine), - utility_assn_eia861=pd.read_sql("utility_assn_eia861", pudl_engine), + out_eia__yearly_utilities=pd.read_sql("out_eia__yearly_utilities", pudl_engine), + core_eia861__yearly_service_territory=pd.read_sql( + "core_eia861__yearly_service_territory", pudl_engine + ), + core_eia861__assn_utility=pd.read_sql("core_eia861__assn_utility", pudl_engine), census_counties=county_gdf, dissolve=dissolve, save_format="geoparquet", diff --git a/src/pudl/analysis/state_demand.py b/src/pudl/analysis/state_demand.py index 936ab0effd..9e74cdc9cc 100644 --- a/src/pudl/analysis/state_demand.py +++ b/src/pudl/analysis/state_demand.py @@ -259,17 +259,17 @@ def load_ventyx_hourly_state_demand(path: str) -> pd.DataFrame: @multi_asset( compute_kind="Python", outs={ - "raw_hourly_demand_matrix_ferc714": AssetOut(), - "utc_offset_ferc714": AssetOut(), + "_out_ferc714__hourly_pivoted_demand_matrix": AssetOut(), + "_out_ferc714__utc_offset": AssetOut(), }, ) def load_hourly_demand_matrix_ferc714( - demand_hourly_pa_ferc714: pd.DataFrame, + core_ferc714__hourly_demand_pa: pd.DataFrame, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Read and format FERC 714 hourly demand into matrix form. Args: - demand_hourly_pa_ferc714: FERC 714 hourly demand time series by planning area. + core_ferc714__hourly_demand_pa: FERC 714 hourly demand time series by planning area. Returns: Hourly demand as a matrix with a `datetime` row index @@ -280,19 +280,22 @@ def load_hourly_demand_matrix_ferc714( of each `respondent_id_ferc714` and reporting `year` (int). """ # Convert UTC to local time (ignoring daylight savings) - demand_hourly_pa_ferc714["utc_offset"] = demand_hourly_pa_ferc714["timezone"].map( - STANDARD_UTC_OFFSETS - ) - demand_hourly_pa_ferc714["datetime"] = utc_to_local( - demand_hourly_pa_ferc714["utc_datetime"], demand_hourly_pa_ferc714["utc_offset"] + core_ferc714__hourly_demand_pa["utc_offset"] = core_ferc714__hourly_demand_pa[ + "timezone" + ].map(STANDARD_UTC_OFFSETS) + core_ferc714__hourly_demand_pa["datetime"] = utc_to_local( + core_ferc714__hourly_demand_pa["utc_datetime"], + core_ferc714__hourly_demand_pa["utc_offset"], ) # Pivot to demand matrix: timestamps x respondents - matrix = demand_hourly_pa_ferc714.pivot( + matrix = core_ferc714__hourly_demand_pa.pivot( index="datetime", columns="respondent_id_ferc714", values="demand_mwh" ) # List timezone by year for each respondent - demand_hourly_pa_ferc714["year"] = demand_hourly_pa_ferc714["report_date"].dt.year - utc_offset = demand_hourly_pa_ferc714.groupby( + core_ferc714__hourly_demand_pa["year"] = core_ferc714__hourly_demand_pa[ + "report_date" + ].dt.year + utc_offset = core_ferc714__hourly_demand_pa.groupby( ["respondent_id_ferc714", "year"], as_index=False )["utc_offset"].first() return matrix, utc_offset @@ -446,13 +449,13 @@ def melt_ferc714_hourly_demand_matrix( ), }, ) -def clean_hourly_demand_matrix_ferc714( - context, raw_hourly_demand_matrix_ferc714: pd.DataFrame +def _out_ferc714__hourly_demand_matrix( + context, _out_ferc714__hourly_pivoted_demand_matrix: pd.DataFrame ) -> pd.DataFrame: """Cleaned and nulled FERC 714 hourly demand matrix. Args: - raw_hourly_demand_matrix_ferc714: FERC 714 hourly demand data in a matrix form. + _out_ferc714__hourly_pivoted_demand_matrix: FERC 714 hourly demand data in a matrix form. Returns: df: Matrix with nulled anomalous values, where respondent-years with too few responses @@ -460,7 +463,7 @@ def clean_hourly_demand_matrix_ferc714( """ min_data = context.op_config["min_data"] min_data_fraction = context.op_config["min_data_fraction"] - df = clean_ferc714_hourly_demand_matrix(raw_hourly_demand_matrix_ferc714) + df = clean_ferc714_hourly_demand_matrix(_out_ferc714__hourly_pivoted_demand_matrix) df = filter_ferc714_hourly_demand_matrix( df, min_data=min_data, min_data_fraction=min_data_fraction ) @@ -468,8 +471,9 @@ def clean_hourly_demand_matrix_ferc714( @asset(compute_kind="Python") -def imputed_hourly_demand_ferc714( - clean_hourly_demand_matrix_ferc714: pd.DataFrame, utc_offset_ferc714: pd.DataFrame +def _out_ferc714__hourly_imputed_demand( + _out_ferc714__hourly_demand_matrix: pd.DataFrame, + _out_ferc714__utc_offset: pd.DataFrame, ) -> pd.DataFrame: """Imputed FERC714 hourly demand in long format. @@ -478,14 +482,14 @@ def imputed_hourly_demand_ferc714( melt data into a long format. Args: - clean_hourly_demand_matrix_ferc714: Cleaned hourly demand matrix from FERC 714. - utc_offset_ferc714: Timezone by year for each respondent. + _out_ferc714__hourly_demand_matrix: Cleaned hourly demand matrix from FERC 714. + _out_ferc714__utc_offset: Timezone by year for each respondent. Returns: df: DataFrame with imputed FERC714 hourly demand. """ - df = impute_ferc714_hourly_demand_matrix(clean_hourly_demand_matrix_ferc714) - df = melt_ferc714_hourly_demand_matrix(df, utc_offset_ferc714) + df = impute_ferc714_hourly_demand_matrix(_out_ferc714__hourly_demand_matrix) + df = melt_ferc714_hourly_demand_matrix(df, _out_ferc714__utc_offset) return df @@ -493,19 +497,19 @@ def imputed_hourly_demand_ferc714( def county_assignments_ferc714( - fipsified_respondents_ferc714, + out_ferc714__respondents_with_fips, ) -> pd.DataFrame: """Load FERC 714 county assignments. Args: - fipsified_respondents_ferc714: From `pudl.output.ferc714`, FERC 714 respondents + out_ferc714__respondents_with_fips: From `pudl.output.ferc714`, FERC 714 respondents with county FIPS IDs. Returns: Dataframe with columns `respondent_id_ferc714`, report `year` (int), and `county_id_fips`. """ - df = fipsified_respondents_ferc714[ + df = out_ferc714__respondents_with_fips[ ["respondent_id_ferc714", "county_id_fips", "report_date"] ] # Drop rows where county is blank or a duplicate @@ -516,7 +520,9 @@ def county_assignments_ferc714( return df -def census_counties(county_censusdp1: gpd.GeoDataFrame) -> gpd.GeoDataFrame: +def census_counties( + core_censusdp1__entity_county: gpd.GeoDataFrame, +) -> gpd.GeoDataFrame: """Load county attributes. Args: @@ -525,7 +531,7 @@ def census_counties(county_censusdp1: gpd.GeoDataFrame) -> gpd.GeoDataFrame: Returns: Dataframe with columns `county_id_fips` and `population`. """ - return county_censusdp1[["geoid10", "dp0010001"]].rename( + return core_censusdp1__entity_county[["geoid10", "dp0010001"]].rename( columns={"geoid10": "county_id_fips", "dp0010001": "population"} ) @@ -534,17 +540,17 @@ def census_counties(county_censusdp1: gpd.GeoDataFrame) -> gpd.GeoDataFrame: def total_state_sales_eia861( - sales_eia861, + core_eia861__yearly_sales, ) -> pd.DataFrame: """Read and format EIA 861 sales by state and year. Args: - sales_eia861: Electricity sales data from EIA 861. + core_eia861__yearly_sales: Electricity sales data from EIA 861. Returns: Dataframe with columns `state_id_fips`, `year`, `demand_mwh`. """ - df = sales_eia861.groupby(["state", "report_date"], as_index=False)[ + df = core_eia861__yearly_sales.groupby(["state", "report_date"], as_index=False)[ "sales_mwh" ].sum() # Convert report_date to year @@ -573,23 +579,22 @@ def total_state_sales_eia861( ), }, ) -def predicted_state_hourly_demand( +def out_ferc714__hourly_predicted_state_demand( context, - imputed_hourly_demand_ferc714: pd.DataFrame, - county_censusdp1: pd.DataFrame, - fipsified_respondents_ferc714: pd.DataFrame, - sales_eia861: pd.DataFrame = None, + _out_ferc714__hourly_imputed_demand: pd.DataFrame, + core_censusdp1__entity_county: pd.DataFrame, + out_ferc714__respondents_with_fips: pd.DataFrame, + core_eia861__yearly_sales: pd.DataFrame = None, ) -> pd.DataFrame: """Predict state hourly demand. Args: - imputed_hourly_demand_ferc714: Hourly demand timeseries, with columns - ``respondent_id_ferc714``, report ``year``, ``utc_datetime``, and - ``demand_mwh``. - county_censusdp1: The county layer of the Census DP1 shapefile. - fipsified_respondents_ferc714: Annual respondents with the county FIPS IDs + _out_ferc714__hourly_imputed_demand: Hourly demand timeseries, with columns + `respondent_id_ferc714`, report `year`, `utc_datetime`, and `demand_mwh`. + core_censusdp1__entity_county: The county layer of the Census DP1 shapefile. + out_ferc714__respondents_with_fips: Annual respondents with the county FIPS IDs for their service territories. - sales_eia861: EIA 861 sales data. If provided, the predicted hourly demand is + core_eia861__yearly_sales: EIA 861 sales data. If provided, the predicted hourly demand is scaled to match these totals. Returns: @@ -600,13 +605,15 @@ def predicted_state_hourly_demand( mean_overlaps = context.op_config["mean_overlaps"] # Call necessary functions - count_assign_ferc714 = county_assignments_ferc714(fipsified_respondents_ferc714) - counties = census_counties(county_censusdp1) - total_sales_eia861 = total_state_sales_eia861(sales_eia861) + count_assign_ferc714 = county_assignments_ferc714( + out_ferc714__respondents_with_fips + ) + counties = census_counties(core_censusdp1__entity_county) + total_sales_eia861 = total_state_sales_eia861(core_eia861__yearly_sales) # Pre-compute list of respondent-years with demand with_demand = ( - imputed_hourly_demand_ferc714.groupby( + _out_ferc714__hourly_imputed_demand.groupby( ["respondent_id_ferc714", "year"], as_index=False )["demand_mwh"] .sum() @@ -640,7 +647,7 @@ def predicted_state_hourly_demand( )["weight"].sum() # Multiply respondent-state weights with demands df = weights.merge( - imputed_hourly_demand_ferc714, on=["respondent_id_ferc714", "year"] + _out_ferc714__hourly_imputed_demand, on=["respondent_id_ferc714", "year"] ) df["demand_mwh"] *= df["weight"] # Scale estimates using state totals diff --git a/src/pudl/etl/__init__.py b/src/pudl/etl/__init__.py index b823aa5909..cade85ebe7 100644 --- a/src/pudl/etl/__init__.py +++ b/src/pudl/etl/__init__.py @@ -32,26 +32,30 @@ logger = pudl.logging_helpers.get_logger(__name__) default_assets = ( - *load_assets_from_modules([eia_bulk_elec_assets], group_name="eia_bulk_elec"), - *load_assets_from_modules([epacems_assets], group_name="epacems"), + *load_assets_from_modules([eia_bulk_elec_assets], group_name="core_eia_bulk_elec"), + *load_assets_from_modules([epacems_assets], group_name="core_epacems"), *load_assets_from_modules([pudl.extract.eia860], group_name="raw_eia860"), *load_assets_from_modules([pudl.transform.eia860], group_name="_core_eia860"), *load_assets_from_modules([pudl.extract.eia861], group_name="raw_eia861"), - *load_assets_from_modules([pudl.transform.eia861], group_name="clean_eia861"), + *load_assets_from_modules( + [pudl.transform.eia861], group_name="core_eia861" + ), # TODO: move one _core asset to separate module? *load_assets_from_modules([pudl.extract.eia923], group_name="raw_eia923"), *load_assets_from_modules([pudl.transform.eia923], group_name="_core_eia923"), - *load_assets_from_modules([pudl.transform.eia], group_name="norm_eia"), + *load_assets_from_modules([pudl.transform.eia], group_name="core_eia"), *load_assets_from_modules([pudl.extract.ferc1], group_name="raw_ferc1"), - *load_assets_from_modules([pudl.transform.ferc1], group_name="norm_ferc1"), + *load_assets_from_modules([pudl.transform.ferc1], group_name="core_ferc1"), *load_assets_from_modules([pudl.extract.ferc714], group_name="raw_ferc714"), - *load_assets_from_modules([pudl.transform.ferc714], group_name="clean_ferc714"), - *load_assets_from_modules([pudl.output.ferc714], group_name="respondents_ferc714"), + *load_assets_from_modules([pudl.transform.ferc714], group_name="core_ferc714"), + *load_assets_from_modules( + [pudl.output.ferc714], group_name="out_respondents_ferc714" + ), *load_assets_from_modules( [pudl.convert.censusdp1tract_to_sqlite, pudl.output.censusdp1tract], - group_name="censusdp1", + group_name="core_censusdp1", ), - *load_assets_from_modules([glue_assets], group_name="glue"), - *load_assets_from_modules([static_assets], group_name="static"), + *load_assets_from_modules([glue_assets], group_name="core_assn"), + *load_assets_from_modules([static_assets], group_name="core_codes"), *load_assets_from_modules( [ pudl.output.eia, @@ -59,22 +63,24 @@ pudl.output.eia923, pudl.output.eia_bulk_elec, ], - group_name="denorm_eia", + group_name="out_eia", + ), + *load_assets_from_modules( + [pudl.analysis.allocate_gen_fuel], group_name="out_allocate_gen_fuel" ), *load_assets_from_modules( - [pudl.analysis.allocate_gen_fuel], group_name="allocate_gen_fuel" + [pudl.analysis.mcoe], group_name="out_derived_gen_attributes" ), - *load_assets_from_modules([pudl.analysis.mcoe], group_name="mcoe"), - *load_assets_from_modules([pudl.output.ferc1], group_name="denorm_ferc1"), + *load_assets_from_modules([pudl.output.ferc1], group_name="out_ferc1"), *load_assets_from_modules( - [pudl.analysis.service_territory], group_name="service_territory_eia861" + [pudl.analysis.service_territory], group_name="out_service_territory_eia861" ), *load_assets_from_modules( - [pudl.analysis.state_demand], group_name="state_demand_ferc714" + [pudl.analysis.state_demand], group_name="out_state_demand_ferc714" ), *load_assets_from_modules( - [pudl.analysis.plant_parts_eia, pudl.analysis.ferc1_eia_record_linkage], - group_name="ferc1_eia_record_linkage", + [pudl.analysis.plant_parts_eia, pudl.analysis.eia_ferc1_record_linkage], + group_name="eia_ferc1_record_linkage", ), ) @@ -113,7 +119,7 @@ def create_non_cems_selection(all_assets: list[AssetsDefinition]) -> AssetSelect all_asset_keys = pudl.helpers.get_asset_keys(all_assets) all_selection = AssetSelection.keys(*all_asset_keys) - cems_selection = AssetSelection.keys(AssetKey("hourly_emissions_epacems")) + cems_selection = AssetSelection.keys(AssetKey("core_epacems__hourly_emissions")) return all_selection - cems_selection.downstream() @@ -147,7 +153,7 @@ def load_dataset_settings_from_file(setting_filename: str) -> dict: name="etl_full_no_cems", selection=create_non_cems_selection(default_assets), description="This job executes all years of all assets except the " - "hourly_emissions_epacems asset and all assets downstream.", + "core_epacems__hourly_emissions asset and all assets downstream.", ), define_asset_job( name="etl_fast", @@ -172,7 +178,7 @@ def load_dataset_settings_from_file(setting_filename: str) -> dict: } }, description="This job executes the most recent year of each asset except the " - "hourly_emissions_epacems asset and all assets downstream.", + "core_epacems__hourly_emissions asset and all assets downstream.", ), ], ) diff --git a/src/pudl/etl/eia_bulk_elec_assets.py b/src/pudl/etl/eia_bulk_elec_assets.py index 419bc770fe..f8cd5fbf69 100644 --- a/src/pudl/etl/eia_bulk_elec_assets.py +++ b/src/pudl/etl/eia_bulk_elec_assets.py @@ -10,7 +10,7 @@ io_manager_key="pudl_sqlite_io_manager", required_resource_keys={"datastore"}, ) -def fuel_receipts_costs_aggs_eia(context): +def core_eia__yearly_fuel_receipts_costs_aggs(context): """Extract and transform EIA bulk electricity aggregates. Returns: diff --git a/src/pudl/etl/epacems_assets.py b/src/pudl/etl/epacems_assets.py index 603ac37fb3..f71e7d47d2 100644 --- a/src/pudl/etl/epacems_assets.py +++ b/src/pudl/etl/epacems_assets.py @@ -1,6 +1,6 @@ """EPA CEMS Hourly Emissions assets. -The :func:`hourly_emissions_epacems` asset defined in this module uses a dagster pattern +The :func:`core_epacems__hourly_emissions` asset defined in this module uses a dagster pattern that is unique from other PUDL assets. The underlying architecture uses ops to create a dynamic graph which is wrapped by a special asset called a graph backed asset that creates an asset @@ -53,23 +53,23 @@ def get_years_from_settings(context): def process_single_year( context, year, - epacamd_eia: pd.DataFrame, - plants_entity_eia: pd.DataFrame, + core_epa__assn_eia_epacamd: pd.DataFrame, + core_eia__entity_plants: pd.DataFrame, ) -> YearPartitions: """Process a single year of EPA CEMS data. Args: context: dagster keyword that provides access to resources and config. year: Year of data to process. - epacamd_eia: The EPA EIA crosswalk table used for harmonizing the + core_epa__assn_eia_epacamd: The EPA EIA crosswalk table used for harmonizing the ORISPL code with EIA. - plants_entity_eia: The EIA Plant entities used for aligning timezones. + core_eia__entity_plants: The EIA Plant entities used for aligning timezones. """ ds = context.resources.datastore epacems_settings = context.resources.dataset_settings.epacems - schema = Resource.from_id("hourly_emissions_epacems").to_pyarrow() - partitioned_path = PudlPaths().output_dir / "hourly_emissions_epacems" + schema = Resource.from_id("core_epacems__hourly_emissions").to_pyarrow() + partitioned_path = PudlPaths().output_dir / "core_epacems__hourly_emissions" partitioned_path.mkdir(exist_ok=True) year_quarters_in_year = { @@ -82,7 +82,9 @@ def process_single_year( logger.info(f"Processing EPA CEMS hourly data for {year_quarter}") df = pudl.extract.epacems.extract(year_quarter=year_quarter, ds=ds) if not df.empty: # If state-year combination has data - df = pudl.transform.epacems.transform(df, epacamd_eia, plants_entity_eia) + df = pudl.transform.epacems.transform( + df, core_epa__assn_eia_epacamd, core_eia__entity_plants + ) table = pa.Table.from_pandas(df, schema=schema, preserve_index=False) # Write to a directory of partitioned parquet files @@ -105,9 +107,9 @@ def consolidate_partitions(context, partitions: list[YearPartitions]) -> None: context: dagster keyword that provides access to resources and config. partitions: Year and state combinations in the output database. """ - partitioned_path = PudlPaths().output_dir / "hourly_emissions_epacems" - monolithic_path = PudlPaths().output_dir / "hourly_emissions_epacems.parquet" - schema = Resource.from_id("hourly_emissions_epacems").to_pyarrow() + partitioned_path = PudlPaths().output_dir / "core_epacems__hourly_emissions" + monolithic_path = PudlPaths().output_dir / "core_epacems__hourly_emissions.parquet" + schema = Resource.from_id("core_epacems__hourly_emissions").to_pyarrow() with pq.ParquetWriter( where=monolithic_path, schema=schema, compression="snappy", version="2.6" @@ -132,8 +134,9 @@ def consolidate_partitions(context, partitions: list[YearPartitions]) -> None: @graph_asset -def hourly_emissions_epacems( - epacamd_eia_unique: pd.DataFrame, plants_entity_eia: pd.DataFrame +def core_epacems__hourly_emissions( + _core_epa__assn_eia_epacamd_unique: pd.DataFrame, + core_eia__entity_plants: pd.DataFrame, ) -> None: """Extract, transform and load CSVs for EPA CEMS. @@ -146,8 +149,8 @@ def hourly_emissions_epacems( partitions = years.map( lambda year: process_single_year( year, - epacamd_eia_unique, - plants_entity_eia, + _core_epa__assn_eia_epacamd_unique, + core_eia__entity_plants, ) ) return consolidate_partitions(partitions.collect()) @@ -155,11 +158,13 @@ def hourly_emissions_epacems( @asset( ins={ - "hourly_emissions_epacems": AssetIn(input_manager_key="epacems_io_manager"), + "core_epacems__hourly_emissions": AssetIn( + input_manager_key="epacems_io_manager" + ), } ) def emissions_unit_ids_epacems( - hourly_emissions_epacems: dd.DataFrame, + core_epacems__hourly_emissions: dd.DataFrame, ) -> pd.DataFrame: """Make unique annual plant_id_eia and emissions_unit_id_epa. @@ -167,7 +172,9 @@ def emissions_unit_ids_epacems( dataframe with unique set of: "plant_id_eia", "year" and "emissions_unit_id_epa" """ epacems_ids = ( - hourly_emissions_epacems[["plant_id_eia", "year", "emissions_unit_id_epa"]] + core_epacems__hourly_emissions[ + ["plant_id_eia", "year", "emissions_unit_id_epa"] + ] .drop_duplicates() .compute() ) diff --git a/src/pudl/etl/glue_assets.py b/src/pudl/etl/glue_assets.py index 6cd4faed92..dbcd1953ea 100644 --- a/src/pudl/etl/glue_assets.py +++ b/src/pudl/etl/glue_assets.py @@ -12,15 +12,15 @@ # TODO (bendnorman): Currently loading all glue tables. Could potentially allow users # to load subsets of the glue tables, see: https://docs.dagster.io/concepts/assets/multi-assets#subsetting-multi-assets # Could split out different types of glue tables into different assets. For example the cross walk table could be a separate asset -# that way dagster doesn't think all glue tables depend on generators_entity_eia, boilers_entity_eia. +# that way dagster doesn't think all glue tables depend on core_eia__entity_generators, core_eia__entity_boilers. @multi_asset( outs={ table_name: AssetOut(io_manager_key="pudl_sqlite_io_manager") for table_name in Package.get_etl_group_tables("glue") - # do not load epacamd_eia glue assets bc they are stand-alone assets below. - if "epacamd_eia" not in table_name + # do not load core_epa__assn_eia_epacamd glue assets bc they are stand-alone assets below. + if "core_epa__assn_eia_epacamd" not in table_name }, required_resource_keys={"datastore", "dataset_settings"}, ) @@ -29,8 +29,8 @@ def create_glue_tables(context): Args: context: dagster keyword that provides access to resources and config. - generators_entity_eia: Static generator attributes compiled from across the EIA-860 and EIA-923 data. - boilers_entity_eia: boilers_entity_eia. + core_eia__entity_generators: Static generator attributes compiled from across the EIA-860 and EIA-923 data. + core_eia__entity_boilers: core_eia__entity_boilers. Returns: A dictionary of DataFrames whose keys are the names of the corresponding @@ -57,7 +57,7 @@ def create_glue_tables(context): @asset(required_resource_keys={"datastore"}) -def raw_epacamd_eia(context) -> pd.DataFrame: +def raw_pudl__assn_eia_epacamd(context) -> pd.DataFrame: """Extract the EPACAMD-EIA Crosswalk from the Datastore.""" logger.info("Extracting the EPACAMD-EIA crosswalk from Zenodo") csv_map = { @@ -79,11 +79,11 @@ def raw_epacamd_eia(context) -> pd.DataFrame: @asset( required_resource_keys={"dataset_settings"}, io_manager_key="pudl_sqlite_io_manager" ) -def epacamd_eia( +def core_epa__assn_eia_epacamd( context, - raw_epacamd_eia: pd.DataFrame, - generators_entity_eia: pd.DataFrame, - boilers_entity_eia: pd.DataFrame, + raw_pudl__assn_eia_epacamd: pd.DataFrame, + core_eia__entity_generators: pd.DataFrame, + core_eia__entity_boilers: pd.DataFrame, ) -> pd.DataFrame: """Clean up the EPACAMD-EIA Crosswalk file. @@ -142,9 +142,9 @@ def epacamd_eia( match the EIA860 working partitions. This indicates whether or not to restrict the crosswalk data so the tests don't fail on foreign key restraints. - raw_epacamd_eia: The result of running this module's extract() function. - generators_entity_eia: The generators_entity_eia table. - boilers_entity_eia: The boilers_entitiy_eia table. + raw_pudl__assn_eia_epacamd: The result of running this module's extract() function. + core_eia__entity_generators: The core_eia__entity_generator table. + core_eia__entity_boilers: The core_eia__entity_boilerstable. Returns: A dictionary containing the cleaned EPACAMD-EIA crosswalk DataFrame. @@ -163,7 +163,7 @@ def epacamd_eia( # Basic column rename, selection, and dtype alignment. crosswalk_clean = ( - raw_epacamd_eia.pipe(pudl.helpers.simplify_columns) + raw_pudl__assn_eia_epacamd.pipe(pudl.helpers.simplify_columns) .rename(columns=column_rename) .filter(list(column_rename.values())) .pipe( @@ -194,13 +194,13 @@ def epacamd_eia( ) crosswalk_clean = pd.merge( crosswalk_clean, - generators_entity_eia[["plant_id_eia", "generator_id"]], + core_eia__entity_generators[["plant_id_eia", "generator_id"]], on=["plant_id_eia", "generator_id"], how="inner", ) crosswalk_clean = pd.merge( crosswalk_clean, - boilers_entity_eia[["plant_id_eia", "boiler_id"]], + core_eia__entity_boilers[["plant_id_eia", "boiler_id"]], on=["plant_id_eia", "boiler_id"], how="inner", ) @@ -209,10 +209,12 @@ def epacamd_eia( @asset -def epacamd_eia_unique(epacamd_eia: pd.DataFrame) -> pd.DataFrame: - """Intermediate asset that contains all unique epacamd_eia matches. +def _core_epa__assn_eia_epacamd_unique( + core_epa__assn_eia_epacamd: pd.DataFrame, +) -> pd.DataFrame: + """Intermediate asset that contains all unique core_epa__assn_eia_epacamd matches. - The epacamd_eia asset contains crosswalk matches from both 2018 and 2021. This + The core_epa__assn_eia_epacamd asset contains crosswalk matches from both 2018 and 2021. This means there are many duplicate matches found from both years. Several downstream assets expect these matches to be unique, so this asset will drop duplicates to serve as the input to those downstream assets. This asset, however, will not itself @@ -220,27 +222,28 @@ def epacamd_eia_unique(epacamd_eia: pd.DataFrame) -> pd.DataFrame: taking the match from the most recent year (2021). Args: - epacamd_eia: Cleaned crosswalk with duplicate matches. + core_epa__assn_eia_epacamd: Cleaned crosswalk with duplicate matches. Returns: Cleaned crosswalk with duplicates removed. """ # Drop fully duplicated matches - epacamd_eia = epacamd_eia.drop_duplicates( - subset=epacamd_eia.columns.difference(["report_year"]) + core_epa__assn_eia_epacamd = core_epa__assn_eia_epacamd.drop_duplicates( + subset=core_epa__assn_eia_epacamd.columns.difference(["report_year"]) ) # Find mismatches where there are different plant_id_eia values between years for # the same plant_id_epa and emissions_unit_id_epa value. - one_to_many = epacamd_eia.groupby(["plant_id_epa", "emissions_unit_id_epa"]).filter( + one_to_many = core_epa__assn_eia_epacamd.groupby( + ["plant_id_epa", "emissions_unit_id_epa"] + ).filter( lambda x: x.plant_id_eia.nunique() > 1 # noqa: PD101 - and x.report_year.nunique() > 1 # noqa: PD101 + and x.report_year.nunique() > 1 # noqa: PD101) ) - # For each mismatch drop the one from 2018, then drop report_year column - return epacamd_eia.drop(one_to_many[one_to_many.report_year == 2018].index).drop( - ["report_year"], axis=1 - ) + return core_epa__assn_eia_epacamd.drop( + one_to_many[one_to_many.report_year == 2018].index + ).drop(["report_year"], axis=1) def correct_epa_eia_plant_id_mapping(df: pd.DataFrame) -> pd.DataFrame: @@ -260,16 +263,16 @@ def correct_epa_eia_plant_id_mapping(df: pd.DataFrame) -> pd.DataFrame: @asset(io_manager_key="pudl_sqlite_io_manager") -def epacamd_eia_subplant_ids( - epacamd_eia_unique: pd.DataFrame, - generators_eia860: pd.DataFrame, +def core_epa__assn_eia_epacamd_subplant_ids( + _core_epa__assn_eia_epacamd_unique: pd.DataFrame, + core_eia860__scd_generators: pd.DataFrame, emissions_unit_ids_epacems: pd.DataFrame, - boiler_generator_assn_eia860: pd.DataFrame, + core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: """Groups units and generators into unique subplant groups. - This takes :func:`epacamd_eia_unique` as an input because this asset so it doesn't - have to deal with duplicate matches that may be present in the :func:`epacamd_eia` + This takes :func:`_core_epa__assn_eia_epacamd_unique` as an input because this asset so it doesn't + have to deal with duplicate matches that may be present in the :func:`core_epa__assn_eia_epacamd` asset due to its use of multiple years of raw crosswalk outputs. This function consists of three primary parts: @@ -295,9 +298,11 @@ def epacamd_eia_subplant_ids( # functioning (#2535) but when it is, ensure that it gets plugged into the dag # BEFORE this step so the subplant IDs can benefit from the more fleshed out units epacamd_eia_complete = ( - augement_crosswalk_with_generators_eia860(epacamd_eia_unique, generators_eia860) + augement_crosswalk_with_generators_eia860( + _core_epa__assn_eia_epacamd_unique, core_eia860__scd_generators + ) .pipe(augement_crosswalk_with_epacamd_ids, emissions_unit_ids_epacems) - .pipe(augement_crosswalk_with_bga_eia860, boiler_generator_assn_eia860) + .pipe(augement_crosswalk_with_bga_eia860, core_eia860__assn_boiler_generator) ) # use graph analysis to identify subplants subplant_ids = make_subplant_ids(epacamd_eia_complete).pipe( @@ -348,16 +353,16 @@ def epacamd_eia_subplant_ids( def augement_crosswalk_with_generators_eia860( - crosswalk_clean: pd.DataFrame, generators_eia860: pd.DataFrame + crosswalk_clean: pd.DataFrame, core_eia860__scd_generators: pd.DataFrame ) -> pd.DataFrame: """Merge any plants that are missing from the EPA crosswalk but appear in EIA-860. Args: crosswalk_clean: transformed EPA CEMS-EIA crosswalk. - generators_eia860: EIA860 generators table. + core_eia860__scd_generators: EIA860 generators table. """ crosswalk_clean = crosswalk_clean.merge( - generators_eia860[["plant_id_eia", "generator_id"]].drop_duplicates(), + core_eia860__scd_generators[["plant_id_eia", "generator_id"]].drop_duplicates(), how="outer", on=["plant_id_eia", "generator_id"], validate="m:1", @@ -385,11 +390,11 @@ def augement_crosswalk_with_epacamd_ids( def augement_crosswalk_with_bga_eia860( - crosswalk_clean: pd.DataFrame, boiler_generator_assn_eia860: pd.DataFrame + crosswalk_clean: pd.DataFrame, core_eia860__assn_boiler_generator: pd.DataFrame ) -> pd.DataFrame: """Merge all EIA Unit IDs into the crosswalk.""" return crosswalk_clean.merge( - boiler_generator_assn_eia860[ + core_eia860__assn_boiler_generator[ ["plant_id_eia", "generator_id", "unit_id_pudl"] ].drop_duplicates(), how="outer", @@ -407,10 +412,10 @@ def _prep_for_networkx(crosswalk: pd.DataFrame) -> pd.DataFrame: """Make surrogate keys for combustors and generators. Args: - crosswalk (pd.DataFrame): epacamd_eia crosswalk + crosswalk (pd.DataFrame): core_epa__assn_eia_epacamd crosswalk Returns: - pd.DataFrame: copy of epacamd_eia crosswalk with new surrogate ID columns + pd.DataFrame: copy of core_epa__assn_eia_epacamd crosswalk with new surrogate ID columns 'combustor_id' and 'generator_id' """ prepped = crosswalk.copy() @@ -431,11 +436,11 @@ def _subplant_ids_from_prepped_crosswalk(prepped: pd.DataFrame) -> pd.DataFrame: """Use networkx graph analysis to create subplant IDs from crosswalk edge list. Args: - prepped (pd.DataFrame): epacamd_eia crosswalked passed through + prepped (pd.DataFrame): core_epa__assn_eia_epacamd crosswalked passed through _prep_for_networkx() Returns: - pd.DataFrame: copy of epacamd_eia crosswalk plus new column 'global_subplant_id' + pd.DataFrame: copy of core_epa__assn_eia_epacamd crosswalk plus new column 'global_subplant_id' """ graph = nx.from_pandas_edgelist( prepped, @@ -528,8 +533,8 @@ def make_subplant_ids(crosswalk: pd.DataFrame) -> pd.DataFrame: Usage Example: epacems = pudl.output.epacems.epacems(states=['ID']) # small subset for quick test - epacamd_eia = pudl_out.epacamd_eia() - filtered_crosswalk = pudl.analysis.epacamd_eia.filter_crosswalk(epacamd_eia, epacems) + core_epa__assn_eia_epacamd = pudl_out.epacamd_eia() + filtered_crosswalk = pudl.analysis.epacamd_eia.filter_crosswalk(core_epa__assn_eia_epacamd, epacems) crosswalk_with_subplant_ids = make_subplant_ids(filtered_crosswalk) Note that sub-plant ids should be used in conjunction with `plant_id_eia` vs. @@ -537,7 +542,7 @@ def make_subplant_ids(crosswalk: pd.DataFrame) -> pd.DataFrame: the transform process. Args: - crosswalk (pd.DataFrame): The epacamd_eia crosswalk + crosswalk (pd.DataFrame): The core_epa__assn_eia_epacamd crosswalk Returns: pd.DataFrame: An edge list connecting EPA units to EIA generators, with diff --git a/src/pudl/etl/static_assets.py b/src/pudl/etl/static_assets.py index ee64f34dc5..c07b97e3d3 100644 --- a/src/pudl/etl/static_assets.py +++ b/src/pudl/etl/static_assets.py @@ -50,8 +50,10 @@ def static_pudl_tables(context): ds = context.resources.datastore dataset_settings = context.resources.dataset_settings - static_pudl_tables_dict = {"political_subdivisions": POLITICAL_SUBDIVISIONS} - static_pudl_tables_dict["datasources"] = dataset_settings.make_datasources_table(ds) + static_pudl_tables_dict = {"core_pudl__codes_subdivisions": POLITICAL_SUBDIVISIONS} + static_pudl_tables_dict[ + "core_pudl__codes_datasources" + ] = dataset_settings.make_datasources_table(ds) return ( Output(output_name=table_name, value=df) for table_name, df in static_pudl_tables_dict.items() @@ -87,7 +89,7 @@ def static_ferc1_tables(): static_table_dict = _read_static_encoding_tables("static_ferc1") static_table_dict.update( { - "ferc_accounts": FERC_ACCOUNTS[ + "core_ferc__codes_accounts": FERC_ACCOUNTS[ ["ferc_account_id", "ferc_account_description"] ], } diff --git a/src/pudl/extract/epacems.py b/src/pudl/extract/epacems.py index 312bb454eb..ea38f04719 100644 --- a/src/pudl/extract/epacems.py +++ b/src/pudl/extract/epacems.py @@ -14,7 +14,7 @@ Pre-transform, the `plant_id_epa` field is a close but not perfect indicator for `plant_id_eia`. In the raw data it's called `Facility ID` (ORISPL code) but that's not -entirely accurate. The epacamd_eia crosswalk will show that the mapping between +entirely accurate. The core_epa__assn_eia_epacamd crosswalk will show that the mapping between `Facility ID` as it appears in CEMS and the `plant_id_eia` field used in EIA data. Hence, we've called it `plant_id_epa` until it gets transformed into `plant_id_eia` during the transform process with help from the crosswalk. @@ -182,6 +182,6 @@ def extract(year_quarter: str, ds: Datastore) -> pd.DataFrame: # If the requested quarter is not found, return an empty df with expected columns: except KeyError: logger.warning(f"No data found for {year_quarter}. Returning empty dataframe.") - res = Resource.from_id("hourly_emissions_epacems") + res = Resource.from_id("core_epacems__hourly_emissions") df = res.format_df(pd.DataFrame()) return df diff --git a/src/pudl/extract/ferc1.py b/src/pudl/extract/ferc1.py index a7eb666c07..9e04552afa 100644 --- a/src/pudl/extract/ferc1.py +++ b/src/pudl/extract/ferc1.py @@ -100,67 +100,67 @@ logger = pudl.logging_helpers.get_logger(__name__) TABLE_NAME_MAP_FERC1: dict[str, dict[str, str]] = { - "fuel_ferc1": { + "core_ferc1__yearly_steam_plants_fuel_sched402": { "dbf": "f1_fuel", "xbrl": "steam_electric_generating_plant_statistics_large_plants_fuel_statistics_402", }, - "plants_steam_ferc1": { + "core_ferc1__yearly_steam_plants_sched402": { "dbf": "f1_steam", "xbrl": "steam_electric_generating_plant_statistics_large_plants_402", }, - "plants_small_ferc1": { + "core_ferc1__yearly_small_plants_sched410": { "dbf": "f1_gnrt_plant", "xbrl": "generating_plant_statistics_410", }, - "plants_hydro_ferc1": { + "core_ferc1__yearly_hydroelectric_plants_sched406": { "dbf": "f1_hydro", "xbrl": "hydroelectric_generating_plant_statistics_large_plants_406", }, - "plants_pumped_storage_ferc1": { + "core_ferc1__yearly_pumped_storage_plants_sched408": { "dbf": "f1_pumped_storage", "xbrl": "pumped_storage_generating_plant_statistics_large_plants_408", }, - "plant_in_service_ferc1": { + "core_ferc1__yearly_plant_in_service_sched204": { "dbf": "f1_plant_in_srvce", "xbrl": "electric_plant_in_service_204", }, - "purchased_power_ferc1": { + "core_ferc1__yearly_purchased_power_and_exchanges_sched326": { "dbf": "f1_purchased_pwr", "xbrl": "purchased_power_326", }, - "electric_energy_sources_ferc1": { + "core_ferc1__yearly_energy_sources_sched401": { "dbf": "f1_elctrc_erg_acct", "xbrl": "electric_energy_account_401a", }, - "electric_energy_dispositions_ferc1": { + "core_ferc1__yearly_energy_dispositions_sched401": { "dbf": "f1_elctrc_erg_acct", "xbrl": "electric_energy_account_401a", }, - "utility_plant_summary_ferc1": { + "core_ferc1__yearly_utility_plant_summary_sched200": { "dbf": "f1_utltyplnt_smmry", "xbrl": "summary_of_utility_plant_and_accumulated_provisions_for_depreciation_amortization_and_depletion_200", }, - "transmission_statistics_ferc1": { + "core_ferc1__yearly_transmission_lines_sched422": { "dbf": "f1_xmssn_line", "xbrl": "transmission_line_statistics_422", }, - "electric_operating_expenses_ferc1": { + "core_ferc1__yearly_operating_expenses_sched320": { "dbf": "f1_elc_op_mnt_expn", "xbrl": "electric_operations_and_maintenance_expenses_320", }, - "balance_sheet_liabilities_ferc1": { + "core_ferc1__yearly_balance_sheet_liabilities_sched110": { "dbf": "f1_bal_sheet_cr", "xbrl": "comparative_balance_sheet_liabilities_and_other_credits_110", }, - "balance_sheet_assets_ferc1": { + "core_ferc1__yearly_balance_sheet_assets_sched110": { "dbf": "f1_comp_balance_db", "xbrl": "comparative_balance_sheet_assets_and_other_debits_110", }, - "income_statement_ferc1": { + "core_ferc1__yearly_income_statements_sched114": { "dbf": ["f1_income_stmnt", "f1_incm_stmnt_2"], "xbrl": "statement_of_income_114", }, - "retained_earnings_ferc1": { + "core_ferc1__yearly_retained_earnings_sched118": { "dbf": "f1_retained_erng", "xbrl": "retained_earnings_118", }, @@ -168,27 +168,27 @@ "dbf": "f1_retained_erng", "xbrl": "retained_earnings_appropriations_118", }, - "depreciation_amortization_summary_ferc1": { + "core_ferc1__yearly_depreciation_summary_sched336": { "dbf": "f1_dacs_epda", "xbrl": "summary_of_depreciation_and_amortization_charges_section_a_336", }, - "electric_plant_depreciation_changes_ferc1": { + "core_ferc1__yearly_depreciation_changes_sched219": { "dbf": "f1_accumdepr_prvsn", "xbrl": "accumulated_provision_for_depreciation_of_electric_utility_plant_changes_section_a_219", }, - "electric_plant_depreciation_functional_ferc1": { + "core_ferc1__yearly_depreciation_by_function_sched219": { "dbf": "f1_accumdepr_prvsn", "xbrl": "accumulated_provision_for_depreciation_of_electric_utility_plant_functional_classification_section_b_219", }, - "electric_operating_revenues_ferc1": { + "core_ferc1__yearly_operating_revenues_sched300": { "dbf": "f1_elctrc_oper_rev", "xbrl": "electric_operating_revenues_300", }, - "cash_flow_ferc1": { + "core_ferc1__yearly_cash_flows_sched120": { "dbf": "f1_cash_flow", "xbrl": "statement_of_cash_flows_120", }, - "electricity_sales_by_rate_schedule_ferc1": { + "core_ferc1__yearly_sales_by_rate_schedules_sched304": { "dbf": "f1_sales_by_sched", "xbrl": [ "sales_of_electricity_by_rate_schedules_account_440_residential_304", @@ -202,7 +202,7 @@ "sales_of_electricity_by_rate_schedules_account_totals_304", ], }, - "other_regulatory_liabilities_ferc1": { + "core_ferc1__yearly_other_regulatory_liabilities_sched278": { "dbf": "f1_othr_reg_liab", "xbrl": "other_regulatory_liabilities_account_254_278", }, diff --git a/src/pudl/glue/ferc1_eia.py b/src/pudl/glue/ferc1_eia.py index 47a49b7a2f..548d5dc459 100644 --- a/src/pudl/glue/ferc1_eia.py +++ b/src/pudl/glue/ferc1_eia.py @@ -255,11 +255,11 @@ def get_plants_ferc1_raw_job() -> JobDefinition: This job expects ferc1_dbf.sqlite and ferc_xbrl.sqlite databases to be populated. """ plant_tables = [ - "plants_hydro_ferc1", - "plants_small_ferc1", - "plants_pumped_storage_ferc1", - "plants_steam_ferc1", - "fuel_ferc1", # bc it has plants/is associated w/ the steam table + "core_ferc1__yearly_hydroelectric_plants_sched406", + "core_ferc1__yearly_small_plants_sched410", + "core_ferc1__yearly_pumped_storage_plants_sched408", + "core_ferc1__yearly_steam_plants_sched402", + "core_ferc1__yearly_steam_plants_fuel_sched402", # bc it has plants/is associated w/ the steam table ] @asset(ins={table_name: AssetIn() for table_name in plant_tables}) @@ -392,19 +392,21 @@ def label_plants_eia(pudl_out: pudl.output.pudltabl.PudlTabl): def label_utilities_ferc1_dbf( - utilities_ferc1_dbf: pd.DataFrame, util_ids_ferc1_raw_dbf: pd.DataFrame + core_pudl__assn_ferc1_dbf_pudl_utilities: pd.DataFrame, + util_ids_ferc1_raw_dbf: pd.DataFrame, ) -> pd.DataFrame: """Get the DBF FERC1 utilities with their names.""" - return utilities_ferc1_dbf.merge( + return core_pudl__assn_ferc1_dbf_pudl_utilities.merge( util_ids_ferc1_raw_dbf, how="outer", on="utility_id_ferc1_dbf" ) def label_utilities_ferc1_xbrl( - utilities_ferc1_xbrl: pd.DataFrame, util_ids_ferc1_raw_xbrl: pd.DataFrame + core_pudl__assn_ferc1_xbrl_pudl_utilities: pd.DataFrame, + util_ids_ferc1_raw_xbrl: pd.DataFrame, ) -> pd.DataFrame: """Get the XBRL FERC1 utilities with their names.""" - return utilities_ferc1_xbrl.merge( + return core_pudl__assn_ferc1_xbrl_pudl_utilities.merge( util_ids_ferc1_raw_xbrl, how="outer", on="utility_id_ferc1_xbrl" ) @@ -412,7 +414,7 @@ def label_utilities_ferc1_xbrl( def get_utility_most_recent_capacity(pudl_engine) -> pd.DataFrame: """Calculate total generation capacity by utility in most recent reported year.""" gen_caps = pd.read_sql( - "SELECT utility_id_eia, capacity_mw, report_date FROM generators_eia860", + "SELECT utility_id_eia, capacity_mw, report_date FROM core_eia860__scd_generators", con=pudl_engine, parse_dates=["report_date"], ) @@ -516,16 +518,16 @@ def glue(ferc1=False, eia=False): - utilities: Unique id and name for each utility for use across the PUDL DB. - plants: Unique id and name for each plant for use across the PUDL DB. - - utilities_eia: EIA operator ids and names attached to a PUDL + - core_pudl__assn_eia_pudl_utilities: EIA operator ids and names attached to a PUDL utility id. - - plants_eia: EIA plant ids and names attached to a PUDL plant id. + - core_pudl__assn_eia_pudl_plants: EIA plant ids and names attached to a PUDL plant id. - utilities_ferc: FERC respondent ids & names attached to a PUDL utility id. - plants_ferc: A combination of FERC plant names and respondent ids, associated with a PUDL plant ID. This is necessary because FERC does not provide plant ids, so the unique plant identifier is a combination of the respondent id and plant name. - - utility_plant_assn: An association table which describes which plants + - core_pudl__assn_utilities_plants: An association table which describes which plants have relationships with what utilities. If a record exists in this table then combination of PUDL utility id & PUDL plant id does have an association of some kind. The nature of that association is @@ -553,36 +555,36 @@ def glue(ferc1=False, eia=False): # from other tables. plant_map = get_plant_map() - plants_pudl = ( + core_pudl__entity_plants_pudl = ( plant_map.loc[:, ["plant_id_pudl", "plant_name_pudl"]] .drop_duplicates("plant_id_pudl") .dropna(how="all") ) - plants_eia = ( + core_pudl__assn_eia_pudl_plants = ( plant_map.loc[:, ["plant_id_eia", "plant_name_eia", "plant_id_pudl"]] .drop_duplicates("plant_id_eia") .dropna(subset=["plant_id_eia"]) ) - plants_ferc1 = ( + core_pudl__assn_ferc1_pudl_plants = ( plant_map.loc[:, ["plant_name_ferc1", "utility_id_ferc1", "plant_id_pudl"]] .drop_duplicates(["plant_name_ferc1", "utility_id_ferc1"]) .dropna(subset=["utility_id_ferc1", "plant_name_ferc1"]) ) utility_map_pudl = get_utility_map_pudl() - utilities_pudl = ( + core_pudl__entity_utilities_pudl = ( utility_map_pudl.loc[:, ["utility_id_pudl", "utility_name_pudl"]] .drop_duplicates("utility_id_pudl") .dropna(how="all") ) - utilities_eia = ( + core_pudl__assn_eia_pudl_utilities = ( utility_map_pudl.loc[ :, ["utility_id_eia", "utility_name_eia", "utility_id_pudl"] ] .drop_duplicates("utility_id_eia") .dropna(subset=["utility_id_eia"]) ) - utilities_ferc1 = ( + core_pudl__assn_ferc1_pudl_utilities = ( utility_map_pudl.loc[ :, ["utility_id_ferc1", "utility_name_ferc1", "utility_id_pudl"] ] @@ -591,12 +593,12 @@ def glue(ferc1=False, eia=False): ) utility_map_ferc1 = get_utility_map_ferc1() - utilities_ferc1_dbf = ( + core_pudl__assn_ferc1_dbf_pudl_utilities = ( utility_map_ferc1.loc[:, ["utility_id_ferc1", "utility_id_ferc1_dbf"]] .drop_duplicates("utility_id_ferc1_dbf") .dropna(subset=["utility_id_ferc1_dbf"]) ) - utilities_ferc1_xbrl = ( + core_pudl__assn_ferc1_xbrl_pudl_utilities = ( utility_map_ferc1.loc[:, ["utility_id_ferc1", "utility_id_ferc1_xbrl"]] .drop_duplicates("utility_id_ferc1_xbrl") .dropna(subset=["utility_id_ferc1_xbrl"]) @@ -619,16 +621,24 @@ def glue(ferc1=False, eia=False): # Now we can concatenate the two dataframes, and get rid of all the columns # except for plant_id and utility_id (which determine the utility to plant # association), and get rid of any duplicates or lingering NaN values... - utility_plant_assn = pd.concat( + core_pudl__assn_utilities_plants = pd.concat( [ - pd.merge(utilities_eia, plants_utilities_eia, on="utility_id_eia"), - pd.merge(utilities_ferc1, plants_utilities_ferc1, on="utility_id_ferc1"), + pd.merge( + core_pudl__assn_eia_pudl_utilities, + plants_utilities_eia, + on="utility_id_eia", + ), + pd.merge( + core_pudl__assn_ferc1_pudl_utilities, + plants_utilities_ferc1, + on="utility_id_ferc1", + ), ], sort=True, ) - utility_plant_assn = ( - utility_plant_assn.loc[:, ["plant_id_pudl", "utility_id_pudl"]] + core_pudl__assn_utilities_plants = ( + core_pudl__assn_utilities_plants.loc[:, ["plant_id_pudl", "utility_id_pudl"]] .dropna() .drop_duplicates() ) @@ -641,7 +651,12 @@ def glue(ferc1=False, eia=False): # Is this check still meaningful with all the EIA plants and utilities that # we're harvesting IDs for, with no names? for df, df_n in zip( - [plants_eia, plants_ferc1, utilities_eia, utilities_ferc1], + [ + core_pudl__assn_eia_pudl_plants, + core_pudl__assn_ferc1_pudl_plants, + core_pudl__assn_eia_pudl_utilities, + core_pudl__assn_ferc1_pudl_utilities, + ], ["plants_eia", "plants_ferc1", "utilities_eia", "utilities_ferc1"], ): if df[pd.isnull(df).any(axis="columns")].shape[0] > 1: @@ -655,20 +670,20 @@ def glue(ferc1=False, eia=False): # sanity checks to ensure that it's (at least kind of) clean. # INSERT SANITY HERE - # Any FERC respondent_id that appears in plants_ferc1 must also exist in - # utilities_ferc1: + # Any FERC respondent_id that appears in core_pudl__assn_ferc1_pudl_plants must also exist in + # core_pudl__assn_ferc1_pudl_utilities: # INSERT MORE SANITY HERE glue_dfs = { - "plants_pudl": plants_pudl, - "utilities_pudl": utilities_pudl, - "plants_ferc1": plants_ferc1, - "utilities_ferc1": utilities_ferc1, - "utilities_ferc1_dbf": utilities_ferc1_dbf, - "utilities_ferc1_xbrl": utilities_ferc1_xbrl, - "plants_eia": plants_eia, - "utilities_eia": utilities_eia, - "utility_plant_assn": utility_plant_assn, + "core_pudl__entity_plants_pudl": core_pudl__entity_plants_pudl, + "core_pudl__entity_utilities_pudl": core_pudl__entity_utilities_pudl, + "core_pudl__assn_ferc1_pudl_plants": core_pudl__assn_ferc1_pudl_plants, + "core_pudl__assn_ferc1_pudl_utilities": core_pudl__assn_ferc1_pudl_utilities, + "core_pudl__assn_ferc1_dbf_pudl_utilities": core_pudl__assn_ferc1_dbf_pudl_utilities, + "core_pudl__assn_ferc1_xbrl_pudl_utilities": core_pudl__assn_ferc1_xbrl_pudl_utilities, + "core_pudl__assn_eia_pudl_plants": core_pudl__assn_eia_pudl_plants, + "core_pudl__assn_eia_pudl_utilities": core_pudl__assn_eia_pudl_utilities, + "core_pudl__assn_utilities_plants": core_pudl__assn_utilities_plants, } return glue_dfs diff --git a/src/pudl/helpers.py b/src/pudl/helpers.py index 558df24a4b..5a051da8d6 100644 --- a/src/pudl/helpers.py +++ b/src/pudl/helpers.py @@ -480,11 +480,11 @@ def date_merge( Args: left: The left dataframe in the merge. Typically monthly in our use - cases if doing a left merge E.g. ``generation_eia923``. + cases if doing a left merge E.g. ``core_eia923__monthly_generation``. Must contain columns specified by ``left_date_col`` and ``on`` argument. right: The right dataframe in the merge. Typically annual in our uses - cases if doing a left merge E.g. ``generators_eia860``. + cases if doing a left merge E.g. ``core_eia860__scd_generators``. Must contain columns specified by ``right_date_col`` and ``on`` argument. on: The columns to merge on that are shared between both dataframes. Typically ID columns like ``plant_id_eia``, ``generator_id`` @@ -1733,7 +1733,7 @@ def scale_by_ownership( gens: table with records at the generator level and generator attributes to be scaled by ownership, must have columns ``plant_id_eia``, ``generator_id``, and ``report_date`` - own_eia860: the ``ownership_eia860`` table + own_eia860: the ``core_eia860__scd_ownership`` table scale_cols: a list of columns in the generator table to slice by ownership fraction validate: how to validate merging the ownership table onto the diff --git a/src/pudl/io_managers.py b/src/pudl/io_managers.py index 15e79eb2a9..8d4249063c 100644 --- a/src/pudl/io_managers.py +++ b/src/pudl/io_managers.py @@ -872,7 +872,7 @@ def epacems_io_manager( init_context: InitResourceContext, ) -> PandasParquetIOManager: """IO Manager that writes EPA CEMS partitions to individual parquet files.""" - schema = Resource.from_id("hourly_emissions_epacems").to_pyarrow() + schema = Resource.from_id("core_epacems__hourly_emissions").to_pyarrow() return PandasParquetIOManager( base_path=UPath(PudlPaths().output_dir), schema=schema ) diff --git a/src/pudl/metadata/classes.py b/src/pudl/metadata/classes.py index 27f8655bdf..f13670ea46 100644 --- a/src/pudl/metadata/classes.py +++ b/src/pudl/metadata/classes.py @@ -49,6 +49,7 @@ FIELD_METADATA_BY_RESOURCE, ) from pudl.metadata.helpers import ( + JINJA_FILTERS, expand_periodic_column_names, format_errors, groupby_aggregate, @@ -157,10 +158,13 @@ def _get_jinja_environment(template_dir: DirectoryPath = None): path = template_dir / "templates" else: path = Path(__file__).parent.resolve() / "templates" - return jinja2.Environment( + environment = jinja2.Environment( loader=jinja2.FileSystemLoader(path), autoescape=True, ) + for func_name, func in JINJA_FILTERS.items(): + environment.filters[func_name] = func + return environment # ---- Class attribute types ---- # @@ -1856,6 +1860,22 @@ def to_sql( ) return metadata + def get_sorted_resources(self) -> StrictList(Resource): + """Get a list of sorted Resources. + + Currently Resources are listed in reverse alphabetical order based + on their name which results in the following order to promote output + tables to users and push intermediate tables to the bottom of the + docs: output, core, intermediate. + + In the future we might want to have more fine grain control over how + Resources are sorted. + + Returns: + A sorted list of resources. + """ + return sorted(self.resources, key=lambda r: r.name, reverse=True) + class CodeMetadata(PudlMeta): """A list of Encoders for standardizing and documenting categorical codes. @@ -1899,15 +1919,15 @@ class DatasetteMetadata(PudlMeta): """ data_sources: list[DataSource] - resources: list[Resource] = Package.from_resource_ids().resources + resources: list[Resource] = Package.from_resource_ids().get_sorted_resources() xbrl_resources: dict[str, list[Resource]] = {} label_columns: dict[str, str] = { - "plants_entity_eia": "plant_name_eia", - "plants_ferc1": "plant_name_ferc1", - "plants_pudl": "plant_name_pudl", - "utilities_entity_eia": "utility_name_eia", - "utilities_ferc1": "utility_name_ferc1", - "utilities_pudl": "utility_name_pudl", + "core_eia__entity_plants": "plant_name_eia", + "core_pudl__assn_ferc1_pudl_plants": "plant_name_ferc1", + "core_pudl__entity_plants_pudl": "plant_name_pudl", + "core_eia__entity_utilities": "utility_name_eia", + "core_pudl__assn_ferc1_pudl_utilities": "utility_name_ferc1", + "core_pudl__entity_utilities_pudl": "utility_name_pudl", } @classmethod @@ -1981,9 +2001,15 @@ def from_data_source_ids( xbrl_resources=xbrl_resources, ) - def to_yaml(self) -> str: + def to_yaml(self, exclude_intermediate_resources: bool = False) -> None: """Output database, table, and column metadata to YAML file.""" template = _get_jinja_environment().get_template("datasette-metadata.yml.jinja") + if exclude_intermediate_resources: + [ + resource + for resource in self.resources + if not resource.name.startswith("_") + ] rendered = template.render( license=LICENSES["cc-by-4.0"], data_sources=self.data_sources, diff --git a/src/pudl/metadata/codes.py b/src/pudl/metadata/codes.py index eaf6d508aa..c83208bf9f 100644 --- a/src/pudl/metadata/codes.py +++ b/src/pudl/metadata/codes.py @@ -20,7 +20,7 @@ import pandas as pd CODE_METADATA: dict[str, dict[str, Any]] = { - "boiler_status_eia": { + "core_eia__codes_boiler_status": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -71,7 +71,7 @@ }, "ignored_codes": [0, "OC", "T", "0", "df"], }, - "boiler_types_eia": { + "core_eia__codes_boiler_types": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -105,7 +105,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "coalmine_types_eia": { + "core_eia__codes_coalmine_types": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -138,7 +138,7 @@ }, "ignored_codes": [], }, - "environmental_equipment_manufacturers_eia": { + "core_eia__codes_environmental_equipment_manufacturers": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -353,7 +353,7 @@ "code_fixes": {}, "ignored_codes": ["NA", "IN"], }, - "emission_control_equipment_types_eia": { + "core_eia__codes_emission_control_equipment_types": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -442,7 +442,7 @@ }, "ignored_codes": ["HRSG1", "HRSG2", "FGD", "OV"], }, - "firing_types_eia": { + "core_eia__codes_firing_types": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -483,7 +483,7 @@ }, "ignored_codes": [], }, - "nox_compliance_strategies_eia": { + "core_eia__codes_nox_compliance_strategies": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -527,7 +527,7 @@ "code_fixes": {"H2": "H2O", "NH": "NH3", "ST": "STM", "ln": "LN"}, "ignored_codes": ["NA"], }, - "nox_control_status_eia": { + "core_eia__codes_nox_control_status": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -571,7 +571,7 @@ "code_fixes": {}, "ignored_codes": ["NA"], }, - "nox_units_eia": { + "core_eia__codes_nox_units": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -602,7 +602,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "averaging_periods_eia": { + "core_eia__codes_averaging_periods": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -628,7 +628,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "particulate_compliance_strategies_eia": { + "core_eia__codes_particulate_compliance_strategies": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -650,7 +650,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "particulate_units_eia": { + "core_eia__codes_particulate_units": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -690,7 +690,7 @@ "code_fixes": {"DP": "PB"}, "ignored_codes": [], }, - "power_purchase_types_ferc1": { + "core_ferc1__codes_power_purchase_types": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -771,7 +771,7 @@ "0", ], }, - "momentary_interruptions_eia": { + "core_eia__codes_momentary_interruptions": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -890,7 +890,7 @@ }, "ignored_codes": [], }, - "boiler_generator_assn_types_eia": { + "core_eia__codes_boiler_generator_assn_types": { "df": pd.DataFrame( columns=[ "code", @@ -905,7 +905,7 @@ "code_fixes": {"t": "T", "a": "A"}, "ignored_codes": ["1"], }, - "operational_status_eia": { + "core_eia__codes_operational_status": { "df": pd.DataFrame( columns=["code", "label", "description", "operational_status"], data=[ @@ -1035,7 +1035,7 @@ }, "ignored_codes": ["CS"], }, - "energy_sources_eia": { + "core_eia__codes_energy_sources": { "df": pd.DataFrame( columns=[ "code", @@ -1581,7 +1581,7 @@ "ST", ], }, - "fuel_transportation_modes_eia": { + "core_eia__codes_fuel_transportation_modes": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -1647,7 +1647,7 @@ }, "ignored_codes": ["UN"], }, - "fuel_types_aer_eia": { + "core_eia__codes_fuel_types_aer": { "df": pd.DataFrame( columns=["code", "description"], data=[ @@ -1674,7 +1674,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "contract_types_eia": { + "core_eia__codes_contract_types": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -1699,7 +1699,7 @@ "code_fixes": {"N": "NC"}, "ignored_codes": [], }, - "prime_movers_eia": { + "core_eia__codes_prime_movers": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -1776,7 +1776,7 @@ "code_fixes": {"ic": "IC"}, # there is literally one 'ic' from 2002. "ignored_codes": [], }, - "sector_consolidated_eia": { + "core_eia__codes_sector_consolidated": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -1816,7 +1816,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "steam_plant_types_eia": { + "core_eia__codes_steam_plant_types": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -1845,7 +1845,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "reporting_frequencies_eia": { + "core_eia__codes_reporting_frequencies": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -1869,7 +1869,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "data_maturities": { + "core_pudl__codes_data_maturities": { "df": pd.DataFrame( columns=["code", "description"], data=[ @@ -1894,7 +1894,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "balancing_authorities_eia": { + "core_eia__codes_balancing_authorities": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -2144,7 +2144,7 @@ "GRID", # 2022 860m code for plant ID 55328 (code: CSTO for previous years) ], }, - "regulations_eia": { + "core_eia__codes_regulations": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -2168,7 +2168,7 @@ "code_fixes": {"St": "ST"}, "ignored_codes": ["NA", "XX"], }, - "so2_compliance_strategies_eia": { # TO DO: harmonize these columns with envr equip data when integrated. + "core_eia__codes_so2_compliance_strategies": { # TO DO: harmonize these columns with envr equip data when integrated. "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -2223,7 +2223,7 @@ "code_fixes": {"NC": "NP"}, "ignored_codes": ["NA", "DB"], }, - "so2_units_eia": { + "core_eia__codes_so2_units": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -2265,7 +2265,7 @@ "code_fixes": {}, "ignored_codes": [], }, - "mercury_compliance_strategies_eia": { # TO DO: harmonize with 2021 data and equip data (most cols here should move to equip table.) + "core_eia__codes_mercury_compliance_strategies": { # TO DO: harmonize with 2021 data and equip data (most cols here should move to equip table.) "df": pd.DataFrame( columns=["code", "label", "description"], data=[ @@ -2371,7 +2371,7 @@ "code_fixes": {}, "ignored_codes": ["NA", "MC", "NP"], }, - "wet_dry_bottom_eia": { + "core_eia__codes_wet_dry_bottom": { "df": pd.DataFrame( columns=["code", "label", "description"], data=[ diff --git a/src/pudl/metadata/dfs.py b/src/pudl/metadata/dfs.py index 06bf104606..0bc14932a7 100644 --- a/src/pudl/metadata/dfs.py +++ b/src/pudl/metadata/dfs.py @@ -263,7 +263,7 @@ ), ) """Association table describing the many-to-many relationships between plant sectors and -various aggregates in fuel_receipts_costs_aggs_eia.""" +various aggregates in core_eia__yearly_fuel_receipts_costs_aggs.""" EIA_FUEL_AGGREGATE_ASSN = pd.read_csv( @@ -288,7 +288,7 @@ ), ) """Association table describing the many-to-many relationships between fuel types and -various aggregates in fuel_receipts_costs_aggs_eia. +various aggregates in core_eia__yearly_fuel_receipts_costs_aggs. Missing from these aggregates are all the "other" categories of gases: OG, BFG, SGP, SC, PG. But those gases combine for about 0.2% of total MMBTU of reported fuel receipts. diff --git a/src/pudl/metadata/fields.py b/src/pudl/metadata/fields.py index 313c585178..ca0a1cdf9d 100644 --- a/src/pudl/metadata/fields.py +++ b/src/pudl/metadata/fields.py @@ -106,7 +106,7 @@ }, "asset_type": { "type": "string", - "description": "Type of asset being reported to the balance_sheet_assets_ferc1 table.", + "description": "Type of asset being reported to the core_ferc1__yearly_balance_sheet_assets_sched110 table.", }, "associated_combined_heat_power": { "type": "boolean", @@ -719,7 +719,7 @@ "energy_displaced_mwh": {"type": "number", "unit": "MWh"}, "energy_disposition_type": { "type": "string", - "description": "Type of energy disposition reported in the electric_energy_dispositions_ferc1. Dispositions include sales to customers, re-sales of energy, energy used internally, losses, etc.", + "description": "Type of energy disposition reported in the core_ferc1__yearly_energy_dispositions_sched401. Dispositions include sales to customers, re-sales of energy, energy used internally, losses, etc.", }, "energy_efficiency_annual_actual_peak_reduction_mw": { "type": "number", @@ -810,7 +810,7 @@ }, "energy_source_type": { "type": "string", - "description": "Type of energy source reported in the electric_energy_sources_ferc1 table. There are three groups of energy sources: generation, power exchanges and transmission.", + "description": "Type of energy source reported in the core_ferc1__yearly_energy_sources_sched401 table. There are three groups of energy sources: generation, power exchanges and transmission.", }, "energy_storage": { "type": "boolean", @@ -1058,7 +1058,11 @@ "description": "Original fuel from which this refined fuel was derived.", "constraints": { "enum": sorted( - set(CODE_METADATA["energy_sources_eia"]["df"]["fuel_derived_from"]) + set( + CODE_METADATA["core_eia__codes_energy_sources"]["df"][ + "fuel_derived_from" + ] + ) ) }, }, @@ -1074,7 +1078,11 @@ "description": "High level fuel group defined in the 2021-2023 EIA Form 860 instructions, Table 28.", "constraints": { "enum": sorted( - set(CODE_METADATA["energy_sources_eia"]["df"]["fuel_group_eia"]) + set( + CODE_METADATA["core_eia__codes_energy_sources"]["df"][ + "fuel_group_eia" + ] + ) ) }, }, @@ -1089,7 +1097,11 @@ "description": "Physical phase of matter of the fuel.", "constraints": { "enum": sorted( - set(CODE_METADATA["energy_sources_eia"]["df"]["fuel_phase"].dropna()) + set( + CODE_METADATA["core_eia__codes_energy_sources"]["df"][ + "fuel_phase" + ].dropna() + ) ) }, }, @@ -1116,7 +1128,11 @@ "description": "Simplified fuel type code used in PUDL", "constraints": { "enum": sorted( - set(CODE_METADATA["energy_sources_eia"]["df"].fuel_type_code_pudl) + set( + CODE_METADATA["core_eia__codes_energy_sources"][ + "df" + ].fuel_type_code_pudl + ) ) }, }, @@ -1125,7 +1141,11 @@ "description": "Simplified fuel type code used in PUDL", "constraints": { "enum": sorted( - set(CODE_METADATA["energy_sources_eia"]["df"].fuel_type_code_pudl) + set( + CODE_METADATA["core_eia__codes_energy_sources"][ + "df" + ].fuel_type_code_pudl + ) ) }, }, @@ -1134,7 +1154,11 @@ "description": "Simplified fuel type code used in PUDL", "constraints": { "enum": sorted( - set(CODE_METADATA["energy_sources_eia"]["df"].fuel_type_code_pudl) + set( + CODE_METADATA["core_eia__codes_energy_sources"][ + "df" + ].fuel_type_code_pudl + ) ) }, }, @@ -1217,17 +1241,17 @@ "description": "The energy contained in fuel burned, measured in million BTU.", "unit": "MMBtu", }, - "heat_rate_mmbtu_mwh": { + "unit_heat_rate_mmbtu_per_mwh": { "type": "number", "description": "Fuel content per unit of electricity generated. Coming from MCOE calculation.", "unit": "MMBtu_MWh", }, - "heat_rate_mmbtu_mwh_eia": { + "unit_heat_rate_mmbtu_per_mwh_eia": { "type": "number", "description": "Fuel content per unit of electricity generated. Coming from MCOE calculation.", "unit": "MMBtu_MWh", }, - "heat_rate_mmbtu_mwh_ferc1": { + "unit_heat_rate_mmbtu_per_mwh_ferc1": { "type": "number", "description": "Fuel content per unit of electricity generated. Calculated from FERC reported fuel consumption and net generation.", "unit": "MMBtu_MWh", @@ -1300,7 +1324,7 @@ }, "liability_type": { "type": "string", - "description": "Type of liability being reported to the balance_sheet_liabilities_ferc1 table.", + "description": "Type of liability being reported to the core_ferc1__yearly_balance_sheet_liabilities_sched110 table.", }, "license_id_ferc1": { "type": "integer", @@ -1507,7 +1531,7 @@ "description": "Net electricity generation for the specified period in megawatt-hours (MWh).", "unit": "MWh", # TODO: disambiguate as this column means something different in - # generation_fuel_eia923: + # core_eia923__monthly_generation_fuel: # "description": "Net generation, year to date in megawatthours (MWh). This is total electrical output net of station service. In the case of combined heat and power plants, this value is intended to include internal consumption of electricity for the purposes of a production process, as well as power put on the grid.", }, "net_generation_mwh_eia": { @@ -2268,7 +2292,9 @@ "description": "Code that specifies what time period data has to be reported (i.e. monthly data or annual totals) and how often the power plant reports this data to EIA. See reporting_frequencies_eia for more details.", "constraints": { "enum": sorted( - set(CODE_METADATA["reporting_frequencies_eia"]["df"]["code"]) + set( + CODE_METADATA["core_eia__codes_reporting_frequencies"]["df"]["code"] + ) ) }, }, @@ -2782,6 +2808,11 @@ "utility_id_eia": { "type": "integer", "description": "The EIA Utility Identification number.", + # TODO: Disambiguate column name. In some cases this specifically refers to + # the utility which operates a given plant or generator, but comes from the + # same set of IDs as all the utility IDs. + # E.g. in core_eia860__scd_ownership or core_eia860__scd_generators it would be something like: + # "description": "EIA-assigned identification number for the company that is responsible for the day-to-day operations of the generator.", }, "utility_id_ferc1": { "type": "integer", @@ -2811,7 +2842,7 @@ "utility_owned_capacity_mw": {"type": "number", "unit": "MW"}, "utility_plant_asset_type": { "type": "string", - "description": "Type of utility plant asset reported in the utility_plant_summary_ferc1 table. Assets include those leased to others, held for future use, construction work-in-progress and details of accumulated depreciation.", + "description": "Type of utility plant asset reported in the core_ferc1__yearly_utility_plant_summary_sched200 table. Assets include those leased to others, held for future use, construction work-in-progress and details of accumulated depreciation.", }, "utility_pobox": {"type": "string"}, "utility_type": { @@ -2820,7 +2851,7 @@ }, "utility_type_other": { "type": "string", - "description": "Freeform description of type of utility reported in one of the other three other utility_type sections in the utility_plant_summary_ferc1 table. This field is reported only in the DBF reporting years (1994-2020).", + "description": "Freeform description of type of utility reported in one of the other three other utility_type sections in the core_ferc1__yearly_utility_plant_summary_sched200 table. This field is reported only in the DBF reporting years (1994-2020).", }, "variable_peak_pricing": {"type": "boolean"}, "virtual_capacity_mw": {"type": "number", "unit": "MW"}, @@ -2942,7 +2973,7 @@ FIELD_METADATA_BY_RESOURCE: dict[str, dict[str, Any]] = { "sector_consolidated_eia": {"code": {"type": "integer"}}, - "plants_hydro_ferc1": { + "core_ferc1__yearly_hydroelectric_plants_sched406": { "plant_type": { "type": "string", "constraints": { @@ -2960,7 +2991,7 @@ }, } }, - "plants_steam_ferc1": { + "core_ferc1__yearly_steam_plants_sched402": { "plant_type": { "type": "string", "constraints": { @@ -2985,14 +3016,18 @@ }, "plant_parts_eia": { "energy_source_code_1": { - "constraints": {"enum": set(CODE_METADATA["energy_sources_eia"]["df"].code)} + "constraints": { + "enum": set(CODE_METADATA["core_eia__codes_energy_sources"]["df"].code) + } }, "prime_movers_eia": { - "constraints": {"enum": set(CODE_METADATA["prime_movers_eia"]["df"].code)} + "constraints": { + "enum": set(CODE_METADATA["core_eia__codes_prime_movers"]["df"].code) + } }, "technology_description": {"constraints": {"enum": set(TECH_DESCRIPTIONS)}}, }, - "transmission_statistics_ferc1": { + "core_ferc1__yearly_transmission_lines_sched422": { "capex_land": { "description": "Cost of Land and land rights for the transmission line." }, @@ -3009,7 +3044,7 @@ "opex_rents": {"description": "Rent expenses for the transmission line."}, "opex_total": {"description": "Overall expenses for the transmission line."}, }, - "demand_hourly_pa_ferc714": { + "core_ferc714__hourly_demand_pa": { "timezone": { "constraints": { "enum": [ diff --git a/src/pudl/metadata/helpers.py b/src/pudl/metadata/helpers.py index 68e4c3c256..1baced1928 100644 --- a/src/pudl/metadata/helpers.py +++ b/src/pudl/metadata/helpers.py @@ -1,4 +1,5 @@ """Functions for manipulating metadata constants.""" +import re from collections import defaultdict from collections.abc import Callable, Iterable from typing import Any @@ -590,3 +591,29 @@ def groupby_aggregate( # noqa: C901 # Enforce original data types, which nulls and errors may have changed result = result.astype(dtypes, copy=False) return result, reports + + +# --- Jinja Filters --- # + + +def _format_resource_name_cross_ref(resource_name: str) -> str: + """Format resource name as a cross_ref for sphinx docs. + + Sphinx throws an error when creating a cross ref target for + a resource that has a preceeding underscore. It is + also possible for resources to have identical names + when the preceeding underscore is removed. This function + adds a preceeding 'i' to cross ref targets for resources + with preceeding underscores. The 'i' will not be rendered + in the docs, only in the .rst files the hyperlinks. + + Args: + resource_name: the name of the resource. + + Returns: + The name of the resource formatted for a sphinx cross ref. + """ + return re.sub(r"^_.*", lambda x: "i" + x.group(0), resource_name) + + +JINJA_FILTERS = {"format_resource_name_cross_ref": _format_resource_name_cross_ref} diff --git a/src/pudl/metadata/resources/allocate_gen_fuel.py b/src/pudl/metadata/resources/allocate_gen_fuel.py index 5a9380d5c9..b42286345a 100644 --- a/src/pudl/metadata/resources/allocate_gen_fuel.py +++ b/src/pudl/metadata/resources/allocate_gen_fuel.py @@ -8,14 +8,14 @@ "distinguish between primary and secondary energy_sources for generators. " "Net generation is allocated equally between energy source codes, so if a " "plant has multiple generators with the same prime_mover_code but different " - "energy source codes the generation_fuel_eia923 records will be associated " + "energy source codes the core_eia923__monthly_generation_fuel records will be associated " "similarly between these two generators. Allocated net generation will still " "be proportional to each generator's net generation or capacity." ) RESOURCE_METADATA: dict[str, dict[str, Any]] = ( { - f"generation_fuel_by_generator_energy_source_{freq}_eia923": { + f"out_eia923__{freq}_generation_fuel_by_generator_energy_source": { "description": ( f"{freq.title()} estimated net generation and fuel consumption " "associated with each combination of generator, energy source, and " @@ -52,7 +52,7 @@ for freq in AGG_FREQS } | { - f"generation_fuel_by_generator_{freq}_eia923": { + f"out_eia923__{freq}_generation_fuel_by_generator": { "description": ( f"{freq.title()} estimated net generation and fuel consumption by generator. " "Based on allocating net electricity generation and fuel consumption reported " @@ -86,7 +86,7 @@ for freq in AGG_FREQS } | { - "generation_fuel_by_generator_energy_source_owner_yearly_eia923": { + "out_eia923__yearly_generation_fuel_by_generator_energy_source_owner": { "description": ( "Yearly estimated net generation and fuel consumption for each generator, " "associated with each combination of generator, energy source, prime mover, " diff --git a/src/pudl/metadata/resources/eia.py b/src/pudl/metadata/resources/eia.py index db59550bcb..72efd02d5f 100644 --- a/src/pudl/metadata/resources/eia.py +++ b/src/pudl/metadata/resources/eia.py @@ -4,7 +4,7 @@ from pudl.metadata.codes import CODE_METADATA RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "balancing_authorities_eia": { + "core_eia__codes_balancing_authorities": { "description": "A coding table describing balancing authorities in EIA-860 and EIA-923.", "schema": { "fields": ["code", "label", "description"], @@ -12,30 +12,30 @@ "foreign_key_rules": { "fields": [["balancing_authority_code_eia"]], "exclude": [ - "advanced_metering_infrastructure_eia861", - "balancing_authority_eia861", - "compiled_geometry_balancing_authority_eia861", - "demand_response_eia861", - "demand_response_water_heater_eia861", - "dynamic_pricing_eia861", - "energy_efficiency_eia861", - "fipsified_respondents_ferc714", - "net_metering_customer_fuel_class_eia861", - "net_metering_misc_eia861", - "non_net_metering_customer_fuel_class_eia861", - "non_net_metering_misc_eia861", - "reliability_eia861", - "sales_eia861", - "summarized_demand_ferc714", + "core_eia861__yearly_advanced_metering_infrastructure", + "core_eia861__yearly_balancing_authority", + "out_eia861__compiled_geometry_balancing_authorities", + "core_eia861__yearly_demand_response", + "core_eia861__yearly_demand_response_water_heater", + "core_eia861__yearly_dynamic_pricing", + "core_eia861__yearly_energy_efficiency", + "out_ferc714__respondents_with_fips", + "core_eia861__yearly_net_metering_customer_fuel_class", + "core_eia861__yearly_net_metering_misc", + "core_eia861__yearly_non_net_metering_customer_fuel_class", + "core_eia861__yearly_non_net_metering_misc", + "core_eia861__yearly_reliability", + "core_eia861__yearly_sales", + "out_ferc714__summarized_demand", ], }, }, - "encoder": CODE_METADATA["balancing_authorities_eia"], + "encoder": CODE_METADATA["core_eia__codes_balancing_authorities"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "boilers_entity_eia": { + "core_eia__entity_boilers": { "description": "Static boiler attributes compiled from the EIA-860 and EIA-923 data.", "schema": { "fields": [ @@ -51,43 +51,43 @@ "etl_group": "entity_eia", "field_namespace": "eia", }, - "boiler_generator_assn_types_eia": { + "core_eia__codes_boiler_generator_assn_types": { "description": "A coding table describing different types of boiler-generator associations in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["boiler_generator_assn_type_code"]]}, }, - "encoder": CODE_METADATA["boiler_generator_assn_types_eia"], + "encoder": CODE_METADATA["core_eia__codes_boiler_generator_assn_types"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "boiler_status_eia": { + "core_eia__codes_boiler_status": { "description": "A coding table describing different types of boiler status in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["boiler_status"]]}, }, - "encoder": CODE_METADATA["boiler_status_eia"], + "encoder": CODE_METADATA["core_eia__codes_boiler_status"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "boiler_types_eia": { + "core_eia__codes_boiler_types": { "description": "A coding table describing different types of boiler regulatory types in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["boiler_type"]]}, }, - "encoder": CODE_METADATA["boiler_types_eia"], + "encoder": CODE_METADATA["core_eia__codes_boiler_types"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "environmental_equipment_manufacturers_eia": { + "core_eia__codes_environmental_equipment_manufacturers": { "description": "A coding table describing manufacturers of boilers and environmental control equipment in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -99,12 +99,14 @@ ] }, }, - "encoder": CODE_METADATA["environmental_equipment_manufacturers_eia"], + "encoder": CODE_METADATA[ + "core_eia__codes_environmental_equipment_manufacturers" + ], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "emission_control_equipment_types_eia": { + "core_eia__codes_emission_control_equipment_types": { "description": "A coding table describing the type of emissions control equipment installed on a boiler.", "schema": { "fields": ["code", "label", "description"], @@ -113,12 +115,12 @@ "fields": [["emission_control_equipment_type_code"]], }, }, - "encoder": CODE_METADATA["emission_control_equipment_types_eia"], + "encoder": CODE_METADATA["core_eia__codes_emission_control_equipment_types"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "firing_types_eia": { + "core_eia__codes_firing_types": { "description": "A coding table describing different boiler firing types in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -127,12 +129,12 @@ "fields": [["firing_type_1"], ["firing_type_2"], ["firing_type_3"]] }, }, - "encoder": CODE_METADATA["firing_types_eia"], + "encoder": CODE_METADATA["core_eia__codes_firing_types"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "nox_compliance_strategies_eia": { + "core_eia__codes_nox_compliance_strategies": { "description": "A coding table describing different compliance strategies used to control nitrogen oxide in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -151,12 +153,12 @@ ] }, }, - "encoder": CODE_METADATA["nox_compliance_strategies_eia"], + "encoder": CODE_METADATA["core_eia__codes_nox_compliance_strategies"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "nox_control_status_eia": { + "core_eia__codes_nox_control_status": { "description": "A coding table describing the operational status of nitrogen oxide control units associated with boilers in the EIA-860 data.", "schema": { "fields": ["code", "label", "description"], @@ -167,24 +169,24 @@ ] }, }, - "encoder": CODE_METADATA["nox_control_status_eia"], + "encoder": CODE_METADATA["core_eia__codes_nox_control_status"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "nox_units_eia": { + "core_eia__codes_nox_units": { "description": "A coding table describing different units of measurement for nitrogen oxide in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["unit_nox"]]}, }, - "encoder": CODE_METADATA["nox_units_eia"], + "encoder": CODE_METADATA["core_eia__codes_nox_units"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "mercury_compliance_strategies_eia": { + "core_eia__codes_mercury_compliance_strategies": { "description": "A coding table describing different compliance strategies used to control mercury in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -203,12 +205,12 @@ ] }, }, - "encoder": CODE_METADATA["mercury_compliance_strategies_eia"], + "encoder": CODE_METADATA["core_eia__codes_mercury_compliance_strategies"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "averaging_periods_eia": { + "core_eia__codes_averaging_periods": { "description": "A coding table describing the averaging period specified by emissions statutes and regulation for in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -217,12 +219,12 @@ "fields": [["period_nox"], ["period_particulate"], ["period_so2"]] }, }, - "encoder": CODE_METADATA["averaging_periods_eia"], + "encoder": CODE_METADATA["core_eia__codes_averaging_periods"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "particulate_compliance_strategies_eia": { + "core_eia__codes_particulate_compliance_strategies": { "description": "A coding table describing different compliance strategies used to control particulate matter in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -235,24 +237,24 @@ ] }, }, - "encoder": CODE_METADATA["particulate_compliance_strategies_eia"], + "encoder": CODE_METADATA["core_eia__codes_particulate_compliance_strategies"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "particulate_units_eia": { + "core_eia__codes_particulate_units": { "description": "A coding table describing different units of measurement for particulate matter in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["unit_particulate"]]}, }, - "encoder": CODE_METADATA["particulate_units_eia"], + "encoder": CODE_METADATA["core_eia__codes_particulate_units"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "regulations_eia": { + "core_eia__codes_regulations": { "description": "A coding table describing the different levels of statutes and codes under which boilers operate in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -266,12 +268,12 @@ ] }, }, - "encoder": CODE_METADATA["regulations_eia"], + "encoder": CODE_METADATA["core_eia__codes_regulations"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "so2_compliance_strategies_eia": { + "core_eia__codes_so2_compliance_strategies": { "description": "A coding table describing different compliance strategies used to control sulfur dioxide in the EIA-860.", "schema": { "fields": ["code", "label", "description"], @@ -290,84 +292,84 @@ ] }, }, - "encoder": CODE_METADATA["so2_compliance_strategies_eia"], + "encoder": CODE_METADATA["core_eia__codes_so2_compliance_strategies"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "wet_dry_bottom_eia": { + "core_eia__codes_wet_dry_bottom": { "description": "A coding table describing whether boiler has a wet or dry bottom in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["wet_dry_bottom"]]}, }, - "encoder": CODE_METADATA["wet_dry_bottom_eia"], + "encoder": CODE_METADATA["core_eia__codes_wet_dry_bottom"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "so2_units_eia": { + "core_eia__codes_so2_units": { "description": "A coding table describing different units of measurement for sulfur dioxide in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["unit_so2"]]}, }, - "encoder": CODE_METADATA["so2_units_eia"], + "encoder": CODE_METADATA["core_eia__codes_so2_units"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "steam_plant_types_eia": { + "core_eia__codes_steam_plant_types": { "description": "A coding table describing different types of steam plants in the EIA-860.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["steam_plant_type_code"]]}, }, - "encoder": CODE_METADATA["steam_plant_types_eia"], + "encoder": CODE_METADATA["core_eia__codes_steam_plant_types"], "sources": ["eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "reporting_frequencies_eia": { + "core_eia__codes_reporting_frequencies": { "description": "A coding table describing different types of reporting frequencies in plants in the EIA-923.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["reporting_frequency_code"]]}, }, - "encoder": CODE_METADATA["reporting_frequencies_eia"], + "encoder": CODE_METADATA["core_eia__codes_reporting_frequencies"], "sources": ["eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "coalmine_types_eia": { + "core_eia__codes_coalmine_types": { "description": "A coding table describing different types of coalmines reported as fuel sources in the EIA-923.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["mine_type_code"]]}, }, - "encoder": CODE_METADATA["coalmine_types_eia"], + "encoder": CODE_METADATA["core_eia__codes_coalmine_types"], "sources": ["eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "contract_types_eia": { + "core_eia__codes_contract_types": { "description": "A coding table describing the various types of fuel supply contracts reported in EIA-923.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["contract_type_code"]]}, }, - "encoder": CODE_METADATA["contract_types_eia"], + "encoder": CODE_METADATA["core_eia__codes_contract_types"], "sources": ["eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "operational_status_eia": { + "core_eia__codes_operational_status": { "description": "Codes and metadata pertaining to operational status reported to EIA. Compiled from EIA-860 instructions and EIA-923 file layout spreadsheets.", "schema": { "fields": [ @@ -381,12 +383,12 @@ "fields": [["operational_status_code"]], }, }, - "encoder": CODE_METADATA["operational_status_eia"], + "encoder": CODE_METADATA["core_eia__codes_operational_status"], "sources": ["eia860", "eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "data_maturities": { + "core_pudl__codes_data_maturities": { "description": "Level of maturities of data records. Some data sources report less-than-final data. PUDL sometimes includes this data, but use at your own risk.", "schema": { "fields": ["code", "description"], @@ -395,12 +397,12 @@ "fields": [["data_maturity"]], }, }, - "encoder": CODE_METADATA["data_maturities"], + "encoder": CODE_METADATA["core_pudl__codes_data_maturities"], "sources": ["eia860", "eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "energy_sources_eia": { + "core_eia__codes_energy_sources": { "description": "Codes and metadata pertaining to energy sources reported to EIA. Compiled from EIA-860 instructions and EIA-923 file layout spreadsheets.", "schema": { "fields": [ @@ -437,7 +439,7 @@ ], }, }, - "encoder": CODE_METADATA["energy_sources_eia"], + "encoder": CODE_METADATA["core_eia__codes_energy_sources"], "sources": ["eia860", "eia923"], "etl_group": "static_eia", "field_namespace": "eia", @@ -451,7 +453,7 @@ "field_namespace": "eia", "create_database_schema": False, }, - "fuel_transportation_modes_eia": { + "core_eia__codes_fuel_transportation_modes": { "description": "Long descriptions of the fuel transportation modes reported in the EIA-860 and EIA-923.", "schema": { "fields": ["code", "label", "description"], @@ -469,24 +471,24 @@ ] }, }, - "encoder": CODE_METADATA["fuel_transportation_modes_eia"], + "encoder": CODE_METADATA["core_eia__codes_fuel_transportation_modes"], "sources": ["eia860", "eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "fuel_types_aer_eia": { + "core_eia__codes_fuel_types_aer": { "description": "Descriptive labels for aggregated fuel types used in the Annual Energy Review. See EIA-923 Fuel Code table for additional information.", "schema": { "fields": ["code", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["fuel_type_code_aer"]]}, }, - "encoder": CODE_METADATA["fuel_types_aer_eia"], + "encoder": CODE_METADATA["core_eia__codes_fuel_types_aer"], "sources": ["eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "generators_entity_eia": { + "core_eia__entity_generators": { "description": "Static generator attributes compiled from across the EIA-860 and EIA-923 data.", "schema": { "fields": [ @@ -514,28 +516,28 @@ "primary_key": ["plant_id_eia", "generator_id"], "foreign_key_rules": { "fields": [["plant_id_eia", "generator_id"]], - # exclude epacamd_eia_subplant_ids bc there are generator ids in this + # exclude core_epa__assn_eia_epacamd_subplant_ids bc there are generator ids in this # glue table that come only from epacamd - "exclude": ["epacamd_eia_subplant_ids"], + "exclude": ["core_epa__assn_eia_epacamd_subplant_ids"], }, }, "sources": ["eia860", "eia923"], "etl_group": "entity_eia", "field_namespace": "eia", }, - "momentary_interruptions_eia": { + "core_eia__codes_momentary_interruptions": { "description": "A coding table for utility definitions of momentary service interruptions.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["momentary_interruption_definition"]]}, }, - "encoder": CODE_METADATA["momentary_interruptions_eia"], + "encoder": CODE_METADATA["core_eia__codes_momentary_interruptions"], "sources": ["eia861"], "etl_group": "static_eia", "field_namespace": "eia", }, - "plants_eia": { + "core_pudl__assn_eia_pudl_plants": { "description": "Association between EIA Plant IDs and manually assigned PUDL Plant IDs", "schema": { "fields": ["plant_id_eia", "plant_name_eia", "plant_id_pudl"], @@ -545,7 +547,7 @@ "etl_group": "glue", "field_namespace": "eia", }, - "plants_entity_eia": { + "core_eia__entity_plants": { "description": "Static plant attributes, compiled from across all EIA-860 and EIA-923 data.", "schema": { "fields": [ @@ -563,20 +565,23 @@ "primary_key": ["plant_id_eia"], "foreign_key_rules": { "fields": [["plant_id_eia"]], - # Excluding plants_eia because it's static and manually compiled + # Excluding core_pudl__assn_eia_pudl_plants because it's static and manually compiled # so it has plants from *all* years of data, even when only a # restricted set of data is processed, leading to constraint # violations. # See: https://github.com/catalyst-cooperative/pudl/issues/1196 - # Exclude the epacamd_eia_subplant_ids table - "exclude": ["plants_eia", "epacamd_eia_subplant_ids"], + # Exclude the core_epa__assn_eia_epacamd_subplant_ids table + "exclude": [ + "core_pudl__assn_eia_pudl_plants", + "core_epa__assn_eia_epacamd_subplant_ids", + ], }, }, "sources": ["eia860", "eia923"], "etl_group": "entity_eia", "field_namespace": "eia", }, - "prime_movers_eia": { + "core_eia__codes_prime_movers": { "description": "Long descriptions explaining the short prime mover codes reported in the EIA-860 and EIA-923.", "schema": { "fields": ["code", "label", "description"], @@ -588,24 +593,24 @@ ] }, }, - "encoder": CODE_METADATA["prime_movers_eia"], + "encoder": CODE_METADATA["core_eia__codes_prime_movers"], "sources": ["eia923", "eia860"], "etl_group": "static_eia", "field_namespace": "eia", }, - "sector_consolidated_eia": { + "core_eia__codes_sector_consolidated": { "description": "Long descriptions for the EIA consolidated NAICS sector codes. Codes and descriptions taken from the EIA-923 File Layout spreadsheet.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["sector_id_eia"]]}, }, - "encoder": CODE_METADATA["sector_consolidated_eia"], + "encoder": CODE_METADATA["core_eia__codes_sector_consolidated"], "sources": ["eia860", "eia923"], "etl_group": "static_eia", "field_namespace": "eia", }, - "utilities_eia": { + "core_pudl__assn_eia_pudl_utilities": { "description": "Associations between the EIA Utility IDs and the manually assigned PUDL Utility IDs.", "schema": { "fields": ["utility_id_eia", "utility_name_eia", "utility_id_pudl"], @@ -615,7 +620,7 @@ "etl_group": "glue", "field_namespace": "eia", }, - "utilities_entity_eia": { + "core_eia__entity_utilities": { "description": "Static attributes of utilities, compiled from all EIA data.", "schema": { "fields": ["utility_id_eia", "utility_name_eia"], @@ -629,42 +634,42 @@ # See: https://github.com/catalyst-cooperative/pudl/issues/1196 # Excluding EIA-861 because they haven't been harvested/normalized. "exclude": [ - "utilities_eia", - "advanced_metering_infrastructure_eia861", - "balancing_authority_assn_eia861", - "compiled_geometry_utility_eia861", - "demand_response_eia861", - "demand_response_water_heater_eia861", - "demand_side_management_ee_dr_eia861", - "demand_side_management_misc_eia861", - "demand_side_management_sales_eia861", - "distributed_generation_fuel_eia861", - "distributed_generation_misc_eia861", - "distributed_generation_tech_eia861", - "distribution_systems_eia861", - "dynamic_pricing_eia861", - "energy_efficiency_eia861", - "fipsified_respondents_ferc714", - "green_pricing_eia861", - "mergers_eia861", - "net_metering_customer_fuel_class_eia861", - "net_metering_misc_eia861", - "non_net_metering_customer_fuel_class_eia861", - "non_net_metering_misc_eia861", - "operational_data_misc_eia861", - "operational_data_revenue_eia861", - "reliability_eia861", - "sales_eia861", - "service_territory_eia861", - "summarized_demand_ferc714", - "utility_assn_eia861", - "utility_data_misc_eia861", - "utility_data_nerc_eia861", - "utility_data_rto_eia861", + "core_pudl__assn_eia_pudl_utilities", + "core_eia861__yearly_advanced_metering_infrastructure", + "core_eia861__assn_balancing_authority", + "out_eia861__compiled_geometry_utilities", + "core_eia861__yearly_demand_response", + "core_eia861__yearly_demand_response_water_heater", + "core_eia861__yearly_demand_side_management_ee_dr", + "core_eia861__yearly_demand_side_management_misc", + "core_eia861__yearly_demand_side_management_sales", + "core_eia861__yearly_distributed_generation_fuel", + "core_eia861__yearly_distributed_generation_misc", + "core_eia861__yearly_distributed_generation_tech", + "core_eia861__yearly_distribution_systems", + "core_eia861__yearly_dynamic_pricing", + "core_eia861__yearly_energy_efficiency", + "out_ferc714__respondents_with_fips", + "core_eia861__yearly_green_pricing", + "core_eia861__yearly_mergers", + "core_eia861__yearly_net_metering_customer_fuel_class", + "core_eia861__yearly_net_metering_misc", + "core_eia861__yearly_non_net_metering_customer_fuel_class", + "core_eia861__yearly_non_net_metering_misc", + "core_eia861__yearly_operational_data_misc", + "core_eia861__yearly_operational_data_revenue", + "core_eia861__yearly_reliability", + "core_eia861__yearly_sales", + "core_eia861__yearly_service_territory", + "out_ferc714__summarized_demand", + "core_eia861__assn_utility", + "core_eia861__yearly_utility_data_misc", + "core_eia861__yearly_utility_data_nerc", + "core_eia861__yearly_utility_data_rto", # Utility IDs in this table are owners, not operators, and we are - # not yet harvesting owner_utility_id_eia from ownership_eia860. + # not yet harvesting owner_utility_id_eia from core_eia860__scd_ownership. # See https://github.com/catalyst-cooperative/pudl/issues/1393 - "generation_fuel_by_generator_energy_source_owner_yearly_eia923", + "out_eia923__yearly_generation_fuel_by_generator_energy_source_owner", ], }, }, @@ -672,7 +677,7 @@ "etl_group": "entity_eia", "field_namespace": "eia", }, - "denorm_utilities_eia": { + "out_eia__yearly_utilities": { "description": ("Denormalized table containing all EIA utility attributes."), "schema": { "fields": [ @@ -710,7 +715,7 @@ "sources": ["eia860", "eia923"], "etl_group": "outputs", }, - "denorm_plants_eia": { + "out_eia__yearly_plants": { "description": ("Denormalized table containing all EIA plant attributes."), "schema": { "fields": [ @@ -775,7 +780,7 @@ "sources": ["eia860", "eia923"], "etl_group": "outputs", }, - "denorm_plants_utilities_eia": { + "_out_eia__plants_utilities": { "description": ( "Denormalized table containing all plant and utility IDs and names from EIA." ), @@ -796,7 +801,7 @@ "sources": ["eia860", "eia923"], "etl_group": "outputs", }, - "denorm_boilers_eia": { + "out_eia__yearly_boilers": { "description": ("Denormalized table containing all EIA boiler attributes."), "schema": { "fields": [ @@ -914,7 +919,7 @@ "sources": ["eia860", "eia923"], "etl_group": "outputs", }, - "denorm_generators_eia": { + "_out_eia__yearly_generators": { "description": ("Denormalized table containing all EIA generator attributes."), "schema": { "fields": [ diff --git a/src/pudl/metadata/resources/eia860.py b/src/pudl/metadata/resources/eia860.py index cdab8d5539..4dc2119132 100644 --- a/src/pudl/metadata/resources/eia860.py +++ b/src/pudl/metadata/resources/eia860.py @@ -2,7 +2,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "boilers_eia860": { + "core_eia860__scd_boilers": { "description": ( "Annually varying boiler attributes, compiled from across all EIA-860 data." ), @@ -104,13 +104,13 @@ "foreign_key_rules": { "fields": [["plant_id_eia", "boiler_id", "report_date"]], # TODO: Excluding monthly data tables since their report_date - # values don't match up with generators_eia860, which is annual, + # values don't match up with core_eia860__scd_generators, which is annual, # so non-january records violate the constraint. # See: https://github.com/catalyst-cooperative/pudl/issues/1196 "exclude": [ - "boiler_fuel_eia923", - "denorm_boiler_fuel_eia923", - "denorm_boiler_fuel_monthly_eia923", + "core_eia923__monthly_boiler_fuel", + "out_eia923__boiler_fuel", + "out_eia923__monthly_boiler_fuel", ], }, }, @@ -118,7 +118,7 @@ "sources": ["eia860", "eia923"], "etl_group": "eia860", }, - "boiler_generator_assn_eia860": { + "core_eia860__assn_boiler_generator": { "description": ( "Associations between boilers and generators as reported in EIA-860 " "Schedule 6, Part A. Augmented with various heuristics within PUDL." @@ -142,7 +142,7 @@ "sources": ["eia860", "eia923"], "etl_group": "eia860", }, - "generators_eia860": { + "core_eia860__scd_generators": { "description": ( "Annually varying generator attributes compiled from across EIA-860 and " "EIA-923 data." @@ -223,23 +223,23 @@ "foreign_key_rules": { "fields": [["plant_id_eia", "generator_id", "report_date"]], # TODO: Excluding monthly data tables since their report_date - # values don't match up with generators_eia860, which is annual, + # values don't match up with core_eia860__scd_generators, which is annual, # so non-january records violate the constraint. # See: https://github.com/catalyst-cooperative/pudl/issues/1196 "exclude": [ - "boiler_fuel_eia923", - "capacity_factor_by_generator_monthly", - "denorm_generation_eia923", - "denorm_generation_monthly_eia923", - "fuel_cost_by_generator_monthly", - "fuel_receipts_costs_eia923", - "generation_eia923", - "generation_fuel_by_generator_energy_source_monthly_eia923", - "generation_fuel_by_generator_monthly_eia923", - "generation_fuel_eia923", - "heat_rate_by_generator_monthly", - "mcoe_monthly", - "mcoe_generators_monthly", + "core_eia923__monthly_boiler_fuel", + "_out_eia__monthly_capacity_factor_by_generator", + "out_eia923__generation", + "out_eia923__monthly_generation", + "_out_eia__monthly_fuel_cost_by_generator", + "core_eia923__monthly_fuel_receipts_costs", + "core_eia923__monthly_generation", + "out_eia923__monthly_generation_fuel_by_generator_energy_source", + "out_eia923__monthly_generation_fuel_by_generator", + "core_eia923__monthly_generation_fuel", + "_out_eia__monthly_heat_rate_by_generator", + "_out_eia__monthly_derived_generator_attributes", + "out_eia__monthly_generators", ], }, }, @@ -247,7 +247,7 @@ "sources": ["eia860", "eia923"], "etl_group": "eia860", }, - "ownership_eia860": { + "core_eia860__scd_ownership": { "description": ( "Generator Ownership, reported in EIA-860 Schedule 4. Includes only " "jointly or third-party owned generators." @@ -278,7 +278,7 @@ "sources": ["eia860"], "etl_group": "eia860", }, - "denorm_ownership_eia860": { + "out_eia860__yearly_ownership": { "description": ( "Generator Ownership, reported in EIA-860 Schedule 4. Includes only " "jointly or third-party owned generators. Denormalized to include plant " @@ -313,7 +313,7 @@ "sources": ["eia860"], "etl_group": "outputs", }, - "plants_eia860": { + "core_eia860__scd_plants": { "description": ( "Annually varying plant attributes, compiled from across all EIA-860 and " "EIA-923 data." @@ -366,31 +366,31 @@ "foreign_key_rules": { "fields": [["plant_id_eia", "report_date"]], # TODO: Excluding monthly data tables since their report_date - # values don't match up with plants_eia860, which is annual, so + # values don't match up with core_eia860__scd_plants, which is annual, so # non-january records fail. # See: https://github.com/catalyst-cooperative/pudl/issues/1196 "exclude": [ - "boiler_fuel_eia923", - "denorm_boiler_fuel_eia923", - "denorm_boiler_fuel_monthly_eia923", - "denorm_fuel_receipts_costs_eia923", - "denorm_fuel_receipts_costs_monthly_eia923", - "denorm_generation_eia923", - "denorm_generation_monthly_eia923", - "denorm_generation_fuel_combined_eia923", - "denorm_generation_fuel_combined_monthly_eia923", - "generation_fuel_by_generator_energy_source_monthly_eia923", - "generation_fuel_by_generator_monthly_eia923", - "fuel_receipts_costs_eia923", - "generation_eia923", - "generation_fuel_eia923", - "generation_fuel_nuclear_eia923", - "heat_rate_by_unit_monthly", - "heat_rate_by_generator_monthly", - "fuel_cost_by_generator_monthly", - "capacity_factor_by_generator_monthly", - "mcoe_monthly", - "mcoe_generators_monthly", + "core_eia923__monthly_boiler_fuel", + "out_eia923__boiler_fuel", + "out_eia923__monthly_boiler_fuel", + "out_eia923__fuel_receipts_costs", + "out_eia923__monthly_fuel_receipts_costs", + "out_eia923__generation", + "out_eia923__monthly_generation", + "out_eia923__generation_fuel_combined", + "out_eia923__monthly_generation_fuel_combined", + "out_eia923__monthly_generation_fuel_by_generator_energy_source", + "out_eia923__monthly_generation_fuel_by_generator", + "core_eia923__monthly_fuel_receipts_costs", + "core_eia923__monthly_generation", + "core_eia923__monthly_generation_fuel", + "core_eia923__monthly_generation_fuel_nuclear", + "_out_eia__monthly_heat_rate_by_unit", + "_out_eia__monthly_heat_rate_by_generator", + "_out_eia__monthly_fuel_cost_by_generator", + "_out_eia__monthly_capacity_factor_by_generator", + "_out_eia__monthly_derived_generator_attributes", + "out_eia__monthly_generators", ], }, }, @@ -398,7 +398,7 @@ "sources": ["eia860", "eia923"], "etl_group": "eia860", }, - "utilities_eia860": { + "core_eia860__scd_utilities": { "description": ( "Annually varying utility attributes, compiled from all EIA data." ), @@ -437,59 +437,59 @@ ["owner_utility_id_eia", "report_date"], ], # TODO: Excluding monthly data tables since their report_date - # values don't match up with plants_eia860, which is annual, so + # values don't match up with core_eia860__scd_plants, which is annual, so # non-january records fail. # See: https://github.com/catalyst-cooperative/pudl/issues/1196 # NOTE: EIA-861 has not gone through harvesting / normalization yet. "exclude": [ - "advanced_metering_infrastructure_eia861", - "balancing_authority_assn_eia861", - "compiled_geometry_utility_eia861", - "demand_response_eia861", - "demand_response_water_heater_eia861", - "demand_side_management_ee_dr_eia861", - "demand_side_management_misc_eia861", - "demand_side_management_sales_eia861", - "denorm_boiler_fuel_eia923", - "denorm_boiler_fuel_monthly_eia923", - "denorm_fuel_receipts_costs_eia923", - "denorm_fuel_receipts_costs_monthly_eia923", - "denorm_generation_eia923", - "denorm_generation_monthly_eia923", - "denorm_generation_fuel_combined_eia923", - "denorm_generation_fuel_combined_monthly_eia923", - "fuel_cost_by_generator_monthly", - "generation_fuel_by_generator_energy_source_monthly_eia923", - "generation_fuel_by_generator_monthly_eia923", + "core_eia861__yearly_advanced_metering_infrastructure", + "core_eia861__assn_balancing_authority", + "out_eia861__compiled_geometry_utilities", + "core_eia861__yearly_demand_response", + "core_eia861__yearly_demand_response_water_heater", + "core_eia861__yearly_demand_side_management_ee_dr", + "core_eia861__yearly_demand_side_management_misc", + "core_eia861__yearly_demand_side_management_sales", + "out_eia923__boiler_fuel", + "out_eia923__monthly_boiler_fuel", + "out_eia923__fuel_receipts_costs", + "out_eia923__monthly_fuel_receipts_costs", + "out_eia923__generation", + "out_eia923__monthly_generation", + "out_eia923__generation_fuel_combined", + "out_eia923__monthly_generation_fuel_combined", + "_out_eia__monthly_fuel_cost_by_generator", + "out_eia923__monthly_generation_fuel_by_generator_energy_source", + "out_eia923__monthly_generation_fuel_by_generator", # Utility IDs in this table are owners, not operators, and we are - # not yet harvesting owner_utility_id_eia from ownership_eia860. + # not yet harvesting owner_utility_id_eia from core_eia860__scd_ownership. # See https://github.com/catalyst-cooperative/pudl/issues/1393 - "generation_fuel_by_generator_energy_source_owner_yearly_eia923", - "distributed_generation_fuel_eia861", - "distributed_generation_misc_eia861", - "distributed_generation_tech_eia861", - "distribution_systems_eia861", - "dynamic_pricing_eia861", - "energy_efficiency_eia861", - "fipsified_respondents_ferc714", - "green_pricing_eia861", - "mcoe_monthly", - "mcoe_generators_monthly", - "mergers_eia861", - "net_metering_customer_fuel_class_eia861", - "net_metering_misc_eia861", - "non_net_metering_customer_fuel_class_eia861", - "non_net_metering_misc_eia861", - "operational_data_misc_eia861", - "operational_data_revenue_eia861", - "reliability_eia861", - "sales_eia861", - "service_territory_eia861", - "summarized_demand_ferc714", - "utility_assn_eia861", - "utility_data_misc_eia861", - "utility_data_nerc_eia861", - "utility_data_rto_eia861", + "out_eia923__yearly_generation_fuel_by_generator_energy_source_owner", + "core_eia861__yearly_distributed_generation_fuel", + "core_eia861__yearly_distributed_generation_misc", + "core_eia861__yearly_distributed_generation_tech", + "core_eia861__yearly_distribution_systems", + "core_eia861__yearly_dynamic_pricing", + "core_eia861__yearly_energy_efficiency", + "out_ferc714__respondents_with_fips", + "core_eia861__yearly_green_pricing", + "_out_eia__monthly_derived_generator_attributes", + "out_eia__monthly_generators", + "core_eia861__yearly_mergers", + "core_eia861__yearly_net_metering_customer_fuel_class", + "core_eia861__yearly_net_metering_misc", + "core_eia861__yearly_non_net_metering_customer_fuel_class", + "core_eia861__yearly_non_net_metering_misc", + "core_eia861__yearly_operational_data_misc", + "core_eia861__yearly_operational_data_revenue", + "core_eia861__yearly_reliability", + "core_eia861__yearly_sales", + "core_eia861__yearly_service_territory", + "out_ferc714__summarized_demand", + "core_eia861__assn_utility", + "core_eia861__yearly_utility_data_misc", + "core_eia861__yearly_utility_data_nerc", + "core_eia861__yearly_utility_data_rto", ], }, }, @@ -497,7 +497,7 @@ "sources": ["eia860", "eia923"], "etl_group": "eia860", }, - "emissions_control_equipment_eia860": { + "core_eia860__scd_emissions_control_equipment": { "description": ( """The cost, type, operating status, retirement date, and install year of emissions control equipment reported to EIA. Includes control ids for sulfur dioxide @@ -527,7 +527,7 @@ "sources": ["eia860"], "etl_group": "eia860", }, - "denorm_emissions_control_equipment_eia860": { + "out_eia860__yearly_emissions_control_equipment": { "description": ( """The cost, type, operating status, retirement date, and install year of emissions control equipment reported to EIA. Includes control ids for sulfur dioxide @@ -564,7 +564,7 @@ "sources": ["eia860"], "etl_group": "eia860", }, - "boiler_emissions_control_equipment_assn_eia860": { + "core_eia860__assn_yearly_boiler_emissions_control_equipment": { "description": ( """A table that links EIA boiler IDs to emissions control IDs for NOx, SO2, mercury, and particulate monitoring. The relationship between the IDs is sometimes many @@ -591,7 +591,7 @@ "sources": ["eia860"], "etl_group": "eia860", }, - "boiler_cooling_assn_eia860": { + "core_eia860__assn_boiler_cooling": { "description": "A table that links EIA boiler IDs to EIA cooling system IDs.", "schema": { "fields": [ @@ -612,7 +612,7 @@ "sources": ["eia860"], "etl_group": "eia860", }, - "boiler_stack_flue_assn_eia860": { + "core_eia860__assn_boiler_stack_flue": { "description": ( """A table that links EIA boiler IDs to EIA stack and/or flue system IDs.""" diff --git a/src/pudl/metadata/resources/eia861.py b/src/pudl/metadata/resources/eia861.py index 8808cf255c..f1f806cb7a 100644 --- a/src/pudl/metadata/resources/eia861.py +++ b/src/pudl/metadata/resources/eia861.py @@ -2,7 +2,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "advanced_metering_infrastructure_eia861": { + "core_eia861__yearly_advanced_metering_infrastructure": { "description": "The data contain number of meters from automated meter readings (AMR) and advanced metering infrastructure (AMI) by state, sector, and balancing authority. The energy served (in megawatthours) for AMI systems is provided. Form EIA-861 respondents also report the number of standard meters (non AMR/AMI) in their system. Historical Changes: We started collecting the number of standard meters in 2013. The monthly survey collected these data from January 2011 to January 2017.", "schema": { "fields": [ @@ -35,7 +35,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "balancing_authority_assn_eia861": { + "core_eia861__assn_balancing_authority": { "description": "Association table showing which combinations of state, balancing authority, and utilities were observed in the data each year.", "schema": { "fields": [ @@ -55,7 +55,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "balancing_authority_eia861": { + "core_eia861__yearly_balancing_authority": { "description": "Annual entity table for balancing authorities.", "schema": { "fields": [ @@ -73,7 +73,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "demand_response_eia861": { + "core_eia861__yearly_demand_response": { "description": "The data contain energy demand response programs by state, sector, and balancing authority. We collect data for the number of customers enrolled, energy savings, potential and actual peak savings, and associated costs.", "schema": { "fields": [ @@ -104,7 +104,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "demand_response_water_heater_eia861": { + "core_eia861__yearly_demand_response_water_heater": { "description": "The number of grid connected water heaters enrolled in demand response programs.", "schema": { "fields": [ @@ -126,7 +126,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "demand_side_management_ee_dr_eia861": { + "core_eia861__yearly_demand_side_management_ee_dr": { "description": "The data contain energy efficiency incremental data, energy efficiency annual data, load management incremental data, load management annual data, annual costs, and the customer counts of price response and time response programs by sector.", "schema": { "fields": [ @@ -160,7 +160,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "demand_side_management_misc_eia861": { + "core_eia861__yearly_demand_side_management_misc": { "schema": { "fields": [ "energy_savings_estimates_independently_verified", @@ -183,7 +183,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "demand_side_management_sales_eia861": { + "core_eia861__yearly_demand_side_management_sales": { "schema": { "fields": [ "nerc_region", @@ -199,7 +199,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "distributed_generation_fuel_eia861": { + "core_eia861__yearly_distributed_generation_fuel": { "schema": { "fields": [ "estimated_or_actual_fuel_data", @@ -215,7 +215,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "distributed_generation_misc_eia861": { + "core_eia861__yearly_distributed_generation_misc": { "schema": { "fields": [ "backup_capacity_mw", @@ -235,7 +235,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "distributed_generation_tech_eia861": { + "core_eia861__yearly_distributed_generation_tech": { "schema": { "fields": [ "capacity_mw", @@ -251,7 +251,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "distribution_systems_eia861": { + "core_eia861__yearly_distribution_systems": { "description": "The number of distribution circuits and circuits with voltage optimization by state.", "schema": { "fields": [ @@ -269,7 +269,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "dynamic_pricing_eia861": { + "core_eia861__yearly_dynamic_pricing": { "description": "The number of customers enrolled in dynamic pricing programs by state, sector, and balancing authority. Respondents check if one or more customers are enrolled in time-of-use pricing, real time pricing, variable peak pricing, critical peak pricing, and critical peak rebates.", "schema": { "fields": [ @@ -293,7 +293,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "energy_efficiency_eia861": { + "core_eia861__yearly_energy_efficiency": { "description": "Incremental energy savings, peak demand savings, weighted average life cycle, and associated costs for the reporting year and life cycle of energy efficiency programs.", "schema": { "fields": [ @@ -320,7 +320,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "green_pricing_eia861": { + "core_eia861__yearly_green_pricing": { "description": "Green pricing program revenue, sales, and customer count by sector and state.", "schema": { "fields": [ @@ -341,7 +341,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "mergers_eia861": { + "core_eia861__yearly_mergers": { "description": "Information about utility mergers and acquisitions.", "schema": { "fields": [ @@ -365,7 +365,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "net_metering_customer_fuel_class_eia861": { + "core_eia861__yearly_net_metering_customer_fuel_class": { "schema": { "fields": [ "balancing_authority_code_eia", @@ -387,7 +387,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "net_metering_misc_eia861": { + "core_eia861__yearly_net_metering_misc": { "schema": { "fields": [ "balancing_authority_code_eia", @@ -402,7 +402,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "non_net_metering_customer_fuel_class_eia861": { + "core_eia861__yearly_non_net_metering_customer_fuel_class": { "schema": { "fields": [ "balancing_authority_code_eia", @@ -420,7 +420,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "non_net_metering_misc_eia861": { + "core_eia861__yearly_non_net_metering_misc": { "schema": { "fields": [ "backup_capacity_mw", @@ -438,7 +438,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "operational_data_misc_eia861": { + "core_eia861__yearly_operational_data_misc": { "schema": { "fields": [ "consumed_by_facility_mwh", @@ -475,7 +475,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "operational_data_revenue_eia861": { + "core_eia861__yearly_operational_data_revenue": { "schema": { "fields": [ "nerc_region", @@ -491,7 +491,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "reliability_eia861": { + "core_eia861__yearly_reliability": { "schema": { "fields": [ "caidi_w_major_event_days_minus_loss_of_service_minutes", @@ -522,7 +522,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "sales_eia861": { + "core_eia861__yearly_sales": { "description": "Annual electricity sales to ultimate customers broken down by utility, balancing authority, state, and customer class.", "schema": { "fields": [ @@ -556,7 +556,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "service_territory_eia861": { + "core_eia861__yearly_service_territory": { "description": "County FIPS codes for counties composing utility service territories.", "schema": { "fields": [ @@ -580,7 +580,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "utility_assn_eia861": { + "core_eia861__assn_utility": { "description": "Association table indicating which states each utility reported data for by year.", "schema": { "fields": [ @@ -598,7 +598,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "utility_data_misc_eia861": { + "core_eia861__yearly_utility_data_misc": { "schema": { "fields": [ "alternative_fuel_vehicle_2_activity", @@ -626,7 +626,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "utility_data_nerc_eia861": { + "core_eia861__yearly_utility_data_nerc": { "schema": { "fields": [ "nerc_region", @@ -641,7 +641,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "utility_data_rto_eia861": { + "core_eia861__yearly_utility_data_rto": { "schema": { "fields": [ "nerc_region", @@ -656,7 +656,7 @@ "sources": ["eia861"], "etl_group": "eia861", }, - "compiled_geometry_utility_eia861": { + "out_eia861__compiled_geometry_utilities": { "description": "County-level spatial data for EIA861 utilities.", "schema": { "fields": [ @@ -676,7 +676,7 @@ "field_namespace": "eia", "etl_group": "service_territories", }, - "compiled_geometry_balancing_authority_eia861": { + "out_eia861__compiled_geometry_balancing_authorities": { "description": "County-level spatial data for EIA861 balancing authorities.", "schema": { "fields": [ diff --git a/src/pudl/metadata/resources/eia923.py b/src/pudl/metadata/resources/eia923.py index 5de5724284..1db74d85a1 100644 --- a/src/pudl/metadata/resources/eia923.py +++ b/src/pudl/metadata/resources/eia923.py @@ -2,7 +2,7 @@ from typing import Any TABLE_DESCRIPTIONS: dict[str, str] = { - "boiler_fuel_eia923": ( + "core_eia923__monthly_boiler_fuel": ( """EIA-923 Monthly Boiler Fuel Consumption and Emissions, from EIA-923 Schedule 3. Reports the quantity of each type of fuel consumed by each boiler on a monthly basis, as @@ -12,7 +12,7 @@ generation units. This table provides better coverage of the entire fleet of generators than the -``generation_fuel_eia923`` table, but the fuel consumption reported here is not directly +``core_eia923__monthly_generation_fuel`` table, but the fuel consumption reported here is not directly associated with a generator. This complicates the heat rate calculation, since the associations between individual boilers and generators are incomplete and can be complex. @@ -20,7 +20,7 @@ Note that a small number of respondents only report annual fuel consumption, and all of it is reported in December.""" ), - "fuel_receipts_costs_eia923": ( + "core_eia923__monthly_fuel_receipts_costs": ( """Data describing fuel deliveries to power plants, reported in EIA-923 Schedule 2, Part A. Each record describes an individual fuel delivery. There can be multiple deliveries of @@ -45,13 +45,13 @@ Additional data which we haven't yet integrated is available in a similar format from 2002-2008 via the EIA-423, and going back as far as 1972 from the FERC-423.""" ), - "generation_eia923": ( + "core_eia923__monthly_generation": ( """EIA-923 Monthly Generating Unit Net Generation. From EIA-923 Schedule 3. Reports the net electricity generated by each reporting generator on a monthly basis. This is the most granular information we have about how much electricity individual generators are producing, but only about half of all the generation reported in the -``generation_fuel_eia923`` appears in this table due to the different reporting +``core_eia923__monthly_generation_fuel`` appears in this table due to the different reporting requirements imposed on different types and sizes of generators. Whenever possible, we use this generator-level net generation to estimate the heat rates @@ -62,7 +62,7 @@ Note that a small number of respondents only report annual net generation, and all of it is reported in December.""" ), - "generation_fuel_eia923": ( + "core_eia923__monthly_generation_fuel": ( """EIA-923 Monthly Generation and Fuel Consumption Time Series. From EIA-923 Schedule 3. Monthly electricity generation and fuel consumption reported for each combination of @@ -71,7 +71,7 @@ of fuel consumption and electricity generation for the entire generation fleet. We use the primary fuels and prime movers reported for each generator along with their capacities to attribute fuel consumption and generation when it isn't directly reported -in the ``generation_eia923`` and ``boiler_fuel_eia923`` tables in order to calculate +in the ``core_eia923__monthly_generation`` and ``core_eia923__monthly_boiler_fuel`` tables in order to calculate capacity factors, heat rates, and the marginal cost of electricity. The table makes a distinction between all fuel consumed and fuel consumed for @@ -82,7 +82,7 @@ Note that a small number of respondents only report annual fuel consumption and net generation, and all of it is reported in December.""" ), - "generation_fuel_nuclear_eia923": ( + "core_eia923__monthly_generation_fuel_nuclear": ( """EIA-923 Monthly Generation and Fuel Consumption Time Series. From EIA-923 Schedule 3. Monthly electricity generation and fuel consumption reported for each combination of @@ -94,16 +94,16 @@ "generation_fuel_combined_eia923": ( """EIA-923 Monthly Generation and Fuel Consumption Time Series. From EIA-923 Schedule 3. -Denormalized, combined data from the ``generation_fuel_eia923`` and -``generation_fuel_nuclear_eia923`` with nuclear generation aggregated from the nuclear +Denormalized, combined data from the ``core_eia923__monthly_generation_fuel`` and +``core_eia923__monthly_generation_fuel_nuclear`` with nuclear generation aggregated from the nuclear generation unit level up to the plant prime mover level, so as to be compatible with fossil fuel generation data.""" ), } RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "boiler_fuel_eia923": { - "description": TABLE_DESCRIPTIONS["boiler_fuel_eia923"], + "core_eia923__monthly_boiler_fuel": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_boiler_fuel"], "schema": { "fields": [ "plant_id_eia", @@ -130,8 +130,8 @@ "sources": ["eia923"], "etl_group": "eia923", }, - "denorm_boiler_fuel_eia923": { - "description": TABLE_DESCRIPTIONS["boiler_fuel_eia923"], + "out_eia923__boiler_fuel": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_boiler_fuel"], "schema": { "fields": [ "report_date", @@ -165,8 +165,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_boiler_fuel_yearly_eia923": { - "description": TABLE_DESCRIPTIONS["boiler_fuel_eia923"], + "out_eia923__yearly_boiler_fuel": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_boiler_fuel"], "schema": { "fields": [ "report_date", @@ -200,8 +200,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_boiler_fuel_monthly_eia923": { - "description": TABLE_DESCRIPTIONS["boiler_fuel_eia923"], + "out_eia923__monthly_boiler_fuel": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_boiler_fuel"], "schema": { "fields": [ "report_date", @@ -235,7 +235,7 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "coalmine_eia923": { + "core_eia923__entity_coalmine": { "description": ( """Attributes of coal mines reporting deliveries in the Fuel Receipts and Costs table, via EIA-923 Schedule 2, Part C. @@ -269,8 +269,8 @@ "sources": ["eia923"], "etl_group": "eia923", }, - "fuel_receipts_costs_eia923": { - "description": TABLE_DESCRIPTIONS["fuel_receipts_costs_eia923"], + "core_eia923__monthly_fuel_receipts_costs": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_fuel_receipts_costs"], "schema": { "fields": [ "plant_id_eia", @@ -301,8 +301,8 @@ "sources": ["eia923"], "etl_group": "eia923", }, - "denorm_fuel_receipts_costs_eia923": { - "description": TABLE_DESCRIPTIONS["fuel_receipts_costs_eia923"], + "out_eia923__fuel_receipts_costs": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_fuel_receipts_costs"], "schema": { "fields": [ "report_date", @@ -347,8 +347,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_fuel_receipts_costs_yearly_eia923": { - "description": TABLE_DESCRIPTIONS["fuel_receipts_costs_eia923"], + "out_eia923__yearly_fuel_receipts_costs": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_fuel_receipts_costs"], "schema": { "fields": [ "report_date", @@ -378,8 +378,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_fuel_receipts_costs_monthly_eia923": { - "description": TABLE_DESCRIPTIONS["fuel_receipts_costs_eia923"], + "out_eia923__monthly_fuel_receipts_costs": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_fuel_receipts_costs"], "schema": { "fields": [ "report_date", @@ -409,8 +409,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "generation_eia923": { - "description": TABLE_DESCRIPTIONS["generation_eia923"], + "core_eia923__monthly_generation": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_generation"], "schema": { "fields": [ "plant_id_eia", @@ -425,8 +425,8 @@ "sources": ["eia923"], "etl_group": "eia923", }, - "denorm_generation_eia923": { - "description": TABLE_DESCRIPTIONS["generation_eia923"], + "out_eia923__generation": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_generation"], "schema": { "fields": [ "report_date", @@ -447,8 +447,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_generation_monthly_eia923": { - "description": TABLE_DESCRIPTIONS["generation_eia923"], + "out_eia923__monthly_generation": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_generation"], "schema": { "fields": [ "report_date", @@ -469,8 +469,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_generation_yearly_eia923": { - "description": TABLE_DESCRIPTIONS["generation_eia923"], + "out_eia923__yearly_generation": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_generation"], "schema": { "fields": [ "report_date", @@ -491,8 +491,8 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "generation_fuel_eia923": { - "description": TABLE_DESCRIPTIONS["generation_fuel_eia923"], + "core_eia923__monthly_generation_fuel": { + "description": TABLE_DESCRIPTIONS["core_eia923__monthly_generation_fuel"], "schema": { "fields": [ "report_date", @@ -520,7 +520,7 @@ "sources": ["eia923"], "etl_group": "eia923", }, - "denorm_generation_fuel_combined_eia923": { + "out_eia923__generation_fuel_combined": { "description": TABLE_DESCRIPTIONS["generation_fuel_combined_eia923"], "schema": { "fields": [ @@ -554,7 +554,7 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_generation_fuel_combined_yearly_eia923": { + "out_eia923__yearly_generation_fuel_combined": { "description": TABLE_DESCRIPTIONS["generation_fuel_combined_eia923"], "schema": { "fields": [ @@ -587,7 +587,7 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "denorm_generation_fuel_combined_monthly_eia923": { + "out_eia923__monthly_generation_fuel_combined": { "description": TABLE_DESCRIPTIONS["generation_fuel_combined_eia923"], "schema": { "fields": [ @@ -620,8 +620,10 @@ "sources": ["eia923"], "etl_group": "outputs", }, - "generation_fuel_nuclear_eia923": { - "description": TABLE_DESCRIPTIONS["generation_fuel_nuclear_eia923"], + "core_eia923__monthly_generation_fuel_nuclear": { + "description": TABLE_DESCRIPTIONS[ + "core_eia923__monthly_generation_fuel_nuclear" + ], "schema": { "fields": [ "plant_id_eia", diff --git a/src/pudl/metadata/resources/eia_bulk_elec.py b/src/pudl/metadata/resources/eia_bulk_elec.py index e1ba85bf5b..1b59c92c35 100644 --- a/src/pudl/metadata/resources/eia_bulk_elec.py +++ b/src/pudl/metadata/resources/eia_bulk_elec.py @@ -3,7 +3,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "fuel_receipts_costs_aggs_eia": { + "core_eia__yearly_fuel_receipts_costs_aggs": { "description": ( "Aggregated fuel receipts and costs from the EIA bulk electricty data." ), diff --git a/src/pudl/metadata/resources/epacems.py b/src/pudl/metadata/resources/epacems.py index 6057c0c59e..5085f8d142 100644 --- a/src/pudl/metadata/resources/epacems.py +++ b/src/pudl/metadata/resources/epacems.py @@ -2,7 +2,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "hourly_emissions_epacems": { + "core_epacems__hourly_emissions": { "description": "Hourly emissions and plant operational data reported via Continuous Emissions Monitoring Systems as required by 40 CFR Part 75.", "schema": { "fields": [ diff --git a/src/pudl/metadata/resources/ferc1.py b/src/pudl/metadata/resources/ferc1.py index b6b95a72b6..6f64e0ccf9 100644 --- a/src/pudl/metadata/resources/ferc1.py +++ b/src/pudl/metadata/resources/ferc1.py @@ -4,7 +4,7 @@ from pudl.metadata.codes import CODE_METADATA RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "balance_sheet_assets_ferc1": { + "core_ferc1__yearly_balance_sheet_assets_sched110": { "description": "Comparative Balance Sheet (Assets and Other Debits). Schedule 110.", "schema": { "fields": [ @@ -29,7 +29,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "balance_sheet_liabilities_ferc1": { + "core_ferc1__yearly_balance_sheet_liabilities_sched110": { "description": "Comparative balance sheet (liabilities and other credits)", "schema": { "fields": [ @@ -54,7 +54,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "cash_flow_ferc1": { + "core_ferc1__yearly_cash_flows_sched120": { "description": "The structured portion of the FERC1 cash flow table - Schedule 120.", "schema": { "fields": [ @@ -76,7 +76,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "depreciation_amortization_summary_ferc1": { + "core_ferc1__yearly_depreciation_summary_sched336": { "description": ( "Depreciation and Amortization of Electric Plant (Account 403, 404, 405) " "Section A: Summary of depreciation and amortization changes. " @@ -105,7 +105,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "electric_energy_sources_ferc1": { + "core_ferc1__yearly_energy_sources_sched401": { "description": ( "Electric Energy Account, sources only. Schedule 401a. Amount of " "electricity the utility obtained from each of several sources, by year." @@ -129,7 +129,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "electric_energy_dispositions_ferc1": { + "core_ferc1__yearly_energy_dispositions_sched401": { "description": ( "Electric Energy Account, dispositions only. Schedule 401a. Electricity " "utilities delived to end users, internal losses, etc." @@ -153,7 +153,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "electric_operating_expenses_ferc1": { + "core_ferc1__yearly_operating_expenses_sched320": { "description": ( "Operating and maintenance costs associated with producing electricty, " "reported in Schedule 320 of FERC Form 1." @@ -175,7 +175,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "electric_plant_depreciation_changes_ferc1": { + "core_ferc1__yearly_depreciation_changes_sched219": { "description": ( "Accumulated provision for depreciation of electric utility plant " "(Account 108). Schedule 219 Section A: balances and changes during year." @@ -205,7 +205,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "electric_plant_depreciation_functional_ferc1": { + "core_ferc1__yearly_depreciation_by_function_sched219": { "description": ( "Accumulated provision for depreciation of electric utility plant " "(Account 108). Schedule 219 Section B: Functional plant classifications." @@ -235,7 +235,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "electricity_sales_by_rate_schedule_ferc1": { + "core_ferc1__yearly_sales_by_rate_schedules_sched304": { "description": ( """The pre-2021 data in this table (extracted from FoxProDB vs. XBRL) is extremely unstructured. Where the post-2020 data (from XBRL) sorts the data @@ -309,7 +309,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "ferc_accounts": { + "core_ferc__codes_accounts": { "description": "Account numbers from the FERC Uniform System of Accounts for Electric Plant, which is defined in Code of Federal Regulations (CFR) Title 18, Chapter I, Subchapter C, Part 101. (See e.g. https://www.law.cornell.edu/cfr/text/18/part-101).", "schema": { "fields": ["ferc_account_id", "ferc_account_description"], @@ -319,7 +319,7 @@ "etl_group": "static_ferc1", "field_namespace": "ferc1", }, - "fuel_ferc1": { + "core_ferc1__yearly_steam_plants_fuel_sched402": { "description": "Annual fuel cost and quantity for steam plants with a capacity of 25+ MW, internal combustion and gas-turbine plants of 10+ MW, and all nuclear plants. As reported on page 402 of FERC Form 1 and extracted from the f1_fuel table in FERC's FoxPro Database.", "schema": { "fields": [ @@ -340,7 +340,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "income_statement_ferc1": { + "core_ferc1__yearly_income_statements_sched114": { "description": "Statement of Income. Schedule 114.", "schema": { "fields": [ @@ -365,7 +365,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "other_regulatory_liabilities_ferc1": { + "core_ferc1__yearly_other_regulatory_liabilities_sched278": { "description": "Other regulatory liabilities, including rate order docket number.", "schema": { "fields": [ @@ -383,7 +383,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "plant_in_service_ferc1": { + "core_ferc1__yearly_plant_in_service_sched204": { "description": ( "Balances and changes to FERC Electric Plant in Service accounts, as " "reported on FERC Form 1, Schedule 204. Data originally from the " @@ -420,7 +420,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "plants_ferc1": { + "core_pudl__assn_ferc1_pudl_plants": { "description": "FERC 1 Plants and their associated manually assigned PUDL Plant IDs", "schema": { "fields": ["utility_id_ferc1", "plant_name_ferc1", "plant_id_pudl"], @@ -435,7 +435,7 @@ "etl_group": "glue", "field_namespace": "ferc1", }, - "plants_hydro_ferc1": { + "core_ferc1__yearly_hydroelectric_plants_sched406": { "description": ( "Hydroelectric generating plant statistics for large plants. Large plants " "have an installed nameplate capacity of more than 10 MW. As reported on " @@ -487,7 +487,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "plants_pumped_storage_ferc1": { + "core_ferc1__yearly_pumped_storage_plants_sched408": { "description": ( "Generating plant statistics for hydroelectric pumped storage plants with " "an installed nameplate capacity of 10+ MW. As reported in Scheudle 408 of " @@ -543,7 +543,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "plants_small_ferc1": { + "core_ferc1__yearly_small_plants_sched410": { "description": """The generating plant statistics for internal combustion plants, gas turbine-plants, conventional hydro plants, and pumped storage plants with less than 10 MW installed nameplate capacity and steam plants with less than 25 MW @@ -584,7 +584,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "plants_steam_ferc1": { + "core_ferc1__yearly_steam_plants_sched402": { "description": ( "Generating plant statistics for steam plants with a capacity of 25+ MW, " "internal combustion and gas-turbine plants of 10+ MW, and all nuclear " @@ -639,19 +639,19 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "power_purchase_types_ferc1": { + "core_ferc1__codes_power_purchase_types": { "description": "Coding table defining different types of electricity power purchases.", "schema": { "fields": ["code", "label", "description"], "primary_key": ["code"], "foreign_key_rules": {"fields": [["purchase_type_code"]]}, }, - "encoder": CODE_METADATA["power_purchase_types_ferc1"], + "encoder": CODE_METADATA["core_ferc1__codes_power_purchase_types"], "sources": ["ferc1"], "etl_group": "static_ferc1", "field_namespace": "ferc1", }, - "purchased_power_ferc1": { + "core_ferc1__yearly_purchased_power_and_exchanges_sched326": { "description": ( "Purchased Power (Account 555) including power exchanges (transactions " "involving a balancing of debits and credits for energy, capacity, etc.) " @@ -685,7 +685,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "transmission_statistics_ferc1": { + "core_ferc1__yearly_transmission_lines_sched422": { "description": ( "Transmission Line Statistics. Schedule 422 of FERC Form 1. Information " "describing transmission lines, the cost of lines, annual operating and " @@ -718,7 +718,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "utilities_ferc1": { + "core_pudl__assn_ferc1_pudl_utilities": { "description": "This table maps two manually assigned utility IDs: a PUDL ID and a FERC1 ID. The PUDL ID maps EIA and FERC1 utilities. The FERC1 ID maps the older DBF respondent IDs to new XBRL entity IDs. This table is generated from a table stored in the PUDL repository: src/package_data/glue/utility_id_pudl.csv", "schema": { "fields": ["utility_id_ferc1", "utility_name_ferc1", "utility_id_pudl"], @@ -729,7 +729,7 @@ "etl_group": "glue", "field_namespace": "ferc1", }, - "utilities_ferc1_dbf": { + "core_pudl__assn_ferc1_dbf_pudl_utilities": { "description": "This table maps the assign utility ID FERC1 to the native utility ID from the FERC1 DBF inputs - originally reported as respondent_id.", "schema": { "fields": ["utility_id_ferc1", "utility_id_ferc1_dbf"], @@ -739,7 +739,7 @@ "etl_group": "glue", "field_namespace": "ferc1", }, - "utilities_ferc1_xbrl": { + "core_pudl__assn_ferc1_xbrl_pudl_utilities": { "description": "This table maps the assign utility ID FERC1 to the native utility ID from the FERC1 XBRL inputs - originally reported as entity_id.", "schema": { "fields": ["utility_id_ferc1", "utility_id_ferc1_xbrl"], @@ -749,7 +749,7 @@ "etl_group": "glue", "field_namespace": "ferc1", }, - "utility_plant_summary_ferc1": { + "core_ferc1__yearly_utility_plant_summary_sched200": { "description": ( "Summary of utility plant and accumulated provisions for depreciation, " "amortization and depletion of utilty plant assets reported annually at " @@ -777,7 +777,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "retained_earnings_ferc1": { + "core_ferc1__yearly_retained_earnings_sched118": { "description": "Retained Earnings - The structed part of schedule 118.", "schema": { "fields": [ @@ -802,17 +802,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "retained_earnings_appropriations_ferc1": { - "description": "Retained Earnings - some of the unstructed part of schedule 118.", - "schema": { - "fields": ["utility_id_ferc1", "report_year", "utility_type", "record_id"], - }, - "sources": ["ferc1"], - "etl_group": "ferc1_disabled", - "field_namespace": "ferc1", - "create_database_schema": False, - }, - "electric_operating_revenues_ferc1": { + "core_ferc1__yearly_operating_revenues_sched300": { "description": ( "Electric operating revenues - The structed part of schedule 300." "There are a number of revenue_type's that do not have sales_mwh," @@ -842,7 +832,7 @@ "etl_group": "ferc1", "field_namespace": "ferc1", }, - "denorm_plants_utilities_ferc1": { + "_out_ferc1__yearly_plants_utilities": { "description": "Denormalized table that contains FERC plant and utility information.", "schema": { "fields": [ @@ -858,7 +848,7 @@ "etl_group": "outputs", "sources": ["ferc1"], }, - "denorm_balance_sheet_assets_ferc1": { + "out_ferc1__yearly_balance_sheet_assets_sched110": { "description": "Denormalized table that contains FERC balance sheet asset information.", "schema": { "fields": [ @@ -885,7 +875,7 @@ "etl_group": "outputs", "sources": ["ferc1"], }, - "denorm_balance_sheet_liabilities_ferc1": { + "out_ferc1__yearly_balance_sheet_liabilities_sched110": { "description": "Denormalized table that contains FERC balance sheet liability information.", "schema": { "fields": [ @@ -912,7 +902,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_cash_flow_ferc1": { + "out_ferc1__yearly_cash_flows_sched120": { "description": "Denormalized table that contains FERC cash flow information.", "schema": { "fields": [ @@ -936,7 +926,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_depreciation_amortization_summary_ferc1": { + "out_ferc1__yearly_depreciation_summary_sched336": { "description": "Denormalized table that contains FERC depreciation amortization information.", "schema": { "fields": [ @@ -962,7 +952,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_electric_energy_dispositions_ferc1": { + "out_ferc1__yearly_energy_dispositions_sched401": { "description": "Denormalized table that contains FERC electric energy dispositions information.", "schema": { "fields": [ @@ -985,7 +975,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_electric_energy_sources_ferc1": { + "out_ferc1__yearly_energy_sources_sched401": { "description": "Denormalized table that contains FERC electric energy sources information.", "schema": { "fields": [ @@ -1008,7 +998,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_electric_operating_expenses_ferc1": { + "out_ferc1__yearly_operating_expenses_sched320": { "description": "Denormalized table that contains FERC electric operating expense information.", "schema": { "fields": [ @@ -1029,7 +1019,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_electric_operating_revenues_ferc1": { + "out_ferc1__yearly_operating_revenues_sched300": { "description": "Denormalized table that contains FERC electric operating revenue information.", "schema": { "fields": [ @@ -1056,7 +1046,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_electric_plant_depreciation_changes_ferc1": { + "out_ferc1__yearly_depreciation_changes_sched219": { "description": "Denormalized table that contains FERC electric plant depreciation changes information.", "schema": { "fields": [ @@ -1085,7 +1075,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_electric_plant_depreciation_functional_ferc1": { + "out_ferc1__yearly_depreciation_by_function_sched219": { "description": ( "Denormalized accumulated provision for depreciation of electric utility " "plant (Account 108). Schedule 219 Section B: Functional plant classifications." @@ -1117,7 +1107,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_electricity_sales_by_rate_schedule_ferc1": { + "out_ferc1__yearly_sales_by_rate_schedules_sched304": { "description": "Denormalized table that contains FERC electricity sales by rate schedule information.", "schema": { "fields": [ @@ -1140,7 +1130,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_income_statement_ferc1": { + "out_ferc1__yearly_income_statements_sched114": { "description": "Denormalized table that contains FERC income statement information.", "schema": { "fields": [ @@ -1167,7 +1157,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_other_regulatory_liabilities_ferc1": { + "out_ferc1__yearly_other_regulatory_liabilities_sched278": { "description": "Denormalized table that contains FERC other regulatory liabilities information.", "schema": { "fields": [ @@ -1187,7 +1177,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_retained_earnings_ferc1": { + "out_ferc1__yearly_retained_earnings_sched118": { "description": "Denormalized table that contains FERC retained earnings information.", "schema": { "fields": [ @@ -1214,7 +1204,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_transmission_statistics_ferc1": { + "out_ferc1__yearly_transmission_lines_sched422": { "description": "Denormalized table that contains FERC transmission statistics information.", "schema": { "fields": [ @@ -1245,7 +1235,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_utility_plant_summary_ferc1": { + "out_ferc1__yearly_utility_plant_summary_sched200": { "description": "Denormalized table that contains FERC utility plant summary information.", "schema": { "fields": [ @@ -1271,7 +1261,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_plants_steam_ferc1": { + "_out_ferc1__yearly_steam_plants_sched402": { "description": "Denormalized table that contains steam plant information from FERC Form 1.", "schema": { "fields": [ @@ -1339,7 +1329,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_plants_small_ferc1": { + "_out_ferc1__yearly_small_plants_sched410": { "description": "Denormalized table that contains small plant information from FERC Form 1.", "schema": { "fields": [ @@ -1374,7 +1364,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_plants_hydro_ferc1": { + "_out_ferc1__yearly_hydroelectric_plants_sched406": { "description": "Denormalized table that contains small plant information from FERC Form 1.", "schema": { "fields": [ @@ -1429,7 +1419,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_plants_pumped_storage_ferc1": { + "_out_ferc1__yearly_pumped_storage_plants_sched408": { "description": "Denormalized table that contains pumped storage plant information from FERC Form 1.", "schema": { "fields": [ @@ -1488,7 +1478,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_fuel_ferc1": { + "out_ferc1__yearly_steam_plants_fuel_sched402": { "description": "Denormalized table that contains fuel information from FERC Form 1.", "schema": { "fields": [ @@ -1517,7 +1507,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_purchased_power_ferc1": { + "out_ferc1__yearly_purchased_power_and_exchanges_sched326": { "description": "Denormalized table of FERC Form 1 Purchased Power data.", "schema": { "fields": [ @@ -1547,7 +1537,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_plant_in_service_ferc1": { + "out_ferc1__yearly_plant_in_service_sched204": { "description": "Denormalized table of FERC Form 1 Electric Plant in Service data.", "schema": { "fields": [ @@ -1574,7 +1564,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_plants_all_ferc1": { + "out_ferc1__yearly_all_plants": { "description": "Denormalized table combining the steam, small generators, hydro, and pumped storage tables from FERC Form 1.", "schema": { "fields": [ @@ -1663,7 +1653,7 @@ "etl_group": "outputs", "field_namespace": "ferc1", }, - "denorm_fuel_by_plant_ferc1": { + "out_ferc1__yearly_steam_plants_fuel_by_plant_sched402": { "description": "A table summarizing FERC fuel data by plant, using FERC Form 1 data.", "schema": { "fields": [ diff --git a/src/pudl/metadata/resources/ferc1_eia_record_linkage.py b/src/pudl/metadata/resources/ferc1_eia_record_linkage.py index c60ecedf3f..2163307820 100644 --- a/src/pudl/metadata/resources/ferc1_eia_record_linkage.py +++ b/src/pudl/metadata/resources/ferc1_eia_record_linkage.py @@ -2,7 +2,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "plant_parts_eia": { + "out_eia__yearly_plant_parts": { "description": """Output table with the aggregation of all EIA plant parts. For use with matching to FERC 1. Practically speaking, a plant is a collection of generator(s). There are many @@ -54,7 +54,7 @@ "fuel_cost_per_mwh", "fuel_type_code_pudl", "generator_retirement_date", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "installation_year", "net_generation_mwh", "generator_operating_year", @@ -80,7 +80,7 @@ "etl_group": "outputs", "field_namespace": "eia", }, - "mega_generators_eia": { + "out_eia__yearly_generators_by_ownership": { "description": "A mega table of all EIA generators with ownership integrated.", "schema": { "fields": [ @@ -106,7 +106,7 @@ "fuel_cost_from_eiaapi", "fuel_cost_per_mmbtu", "fuel_cost_per_mwh", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "net_generation_mwh", "total_fuel_cost", "total_mmbtu", @@ -122,7 +122,7 @@ "etl_group": "outputs", "field_namespace": "eia", }, - "out__yearly_plants_all_ferc1_plant_parts_eia": { + "out_pudl__yearly_assn_eia_ferc1_plant_parts": { "description": """This table links power plant data reported in FERC Form 1 to related EIA data. It answers the question "What EIA data reported about plants or generators should be associated with a given plant record found in the FERC Form 1." @@ -173,7 +173,7 @@ "capacity_mw_eia", "capacity_factor_eia", "total_mmbtu_eia", - "heat_rate_mmbtu_mwh_eia", + "unit_heat_rate_mmbtu_per_mwh_eia", "fuel_type_code_pudl_eia", "installation_year_eia", "plant_part_id_eia", @@ -256,7 +256,7 @@ "total_fuel_cost_ferc1", "total_mmbtu_ferc1", "fuel_type_code_pudl_ferc1", - "heat_rate_mmbtu_mwh_ferc1", + "unit_heat_rate_mmbtu_per_mwh_ferc1", ], "primary_key": ["record_id_ferc1"], }, diff --git a/src/pudl/metadata/resources/ferc714.py b/src/pudl/metadata/resources/ferc714.py index 2a85f8c6ae..76baba2466 100644 --- a/src/pudl/metadata/resources/ferc714.py +++ b/src/pudl/metadata/resources/ferc714.py @@ -3,7 +3,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "respondent_id_ferc714": { + "core_ferc714__respondent_id": { "description": "Respondent identification. FERC Form 714, Part I, Schedule 1.", "schema": { "fields": [ @@ -18,7 +18,7 @@ "field_namespace": "ferc714", "etl_group": "ferc714", }, - "demand_hourly_pa_ferc714": { + "core_ferc714__hourly_demand_pa": { "description": ( "Hourly electricity demand by planning area. FERC Form 714, Part III, " "Schedule 2a." @@ -37,7 +37,7 @@ "field_namespace": "ferc714", "etl_group": "ferc714", }, - "fipsified_respondents_ferc714": { + "out_ferc714__respondents_with_fips": { "description": ( "Annual respondents with the county FIPS IDs for their service territories." ), @@ -67,7 +67,7 @@ "field_namespace": "ferc714", "etl_group": "outputs", }, - "summarized_demand_ferc714": { + "out_ferc714__summarized_demand": { "description": ( "Compile FERC 714 annualized, categorized respondents and summarize values." ), diff --git a/src/pudl/metadata/resources/glue.py b/src/pudl/metadata/resources/glue.py index 3a4898baf6..35f680bf8c 100644 --- a/src/pudl/metadata/resources/glue.py +++ b/src/pudl/metadata/resources/glue.py @@ -2,7 +2,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "epacamd_eia": { + "core_epa__assn_eia_epacamd": { "description": """This crosswalk table comes from the EPA's Github repo camd-eia-crosswalk: https://github.com/USEPA/camd-eia-crosswalk. @@ -17,7 +17,7 @@ to create a temporal version of the crosswalk at some point. Our version of the crosswalk clarifies some of the column names and removes unmatched -rows. The :func:`pudl.etl.glue_assets.epacamd_eia` function doc strings explain +rows. The :func:`pudl.etl.glue_assets.core_epa__assn_eia_epacamd` function doc strings explain what changes are made from the EPA's version.""", "schema": { "fields": [ @@ -34,8 +34,8 @@ "etl_group": "glue", "sources": ["epacamd_eia"], }, - "epacamd_eia_subplant_ids": { - "description": """This table is an augmented version of the epacamd_eia + "core_epa__assn_eia_epacamd_subplant_ids": { + "description": """This table is an augmented version of the core_epa__assn_eia_epacamd crosswalk table which initally comes from the EPA's Github repo camd-eia-crosswalk: https://github.com/USEPA/camd-eia-crosswalk. It's purpose is to connect EPA units with EIA units, and generators. @@ -54,8 +54,8 @@ make aggregation to the parent entity difficult or inappropriate. This table inherits from the EPA's crosswalk, the ID's from EPA CAMD -hourly_emissions_epacems table itself, the boiler_generator_assn_eia860 table and the -generators_eia860 table. While the epacamd_eia table is the core backbone of the table, +core_epacems__hourly_emissions table itself, the core_eia860__assn_boiler_generator table and the +core_eia860__scd_generators table. While the core_epa__assn_eia_epacamd table is the core backbone of the table, EPA CAMD id's ensure there is complete coverage of EPA CAMD reporting units. The EIA 860 table addition ensures there is also complete coverage of those units as well. @@ -64,7 +64,7 @@ from: https://catalystcoop-pudl.readthedocs.io/en/latest/autoapi/index.html But by analyzing the relationships between combustors and generators, -as provided in the epacamd_eia crosswalk, we can identify distinct power plants. +as provided in the core_epa__assn_eia_epacamd crosswalk, we can identify distinct power plants. These are the smallest coherent units of aggregation. This table does not have primary keys because the primary keys would have been: diff --git a/src/pudl/metadata/resources/mcoe.py b/src/pudl/metadata/resources/mcoe.py index bf1b63fd56..740506d7fd 100644 --- a/src/pudl/metadata/resources/mcoe.py +++ b/src/pudl/metadata/resources/mcoe.py @@ -1,4 +1,4 @@ -"""Resource metadata for the MCOE (marginal cost of electricity) tables.""" +"""Resource metadata for the generator table with derived attributes.""" from typing import Any AGG_FREQS = ["yearly", "monthly"] @@ -11,7 +11,7 @@ ) RESOURCE_METADATA: dict[str, dict[str, Any]] = ( { - f"heat_rate_by_unit_{freq}": { + f"_out_eia__{freq}_heat_rate_by_unit": { "description": ( f"{freq.title()} heat rate estimates by generation unit. Generation " "units are identified by ``unit_id_pudl`` and are composed of a set of " @@ -24,7 +24,7 @@ "unit_id_pudl", "net_generation_mwh", "fuel_consumed_for_electricity_mmbtu", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", ], "primary_key": [ "report_date", @@ -39,7 +39,7 @@ for freq in AGG_FREQS } | { - f"heat_rate_by_generator_{freq}": { + f"_out_eia__{freq}_heat_rate_by_generator": { "description": ( f"{freq.title()} heat rate estimates by generator. These are actually " "just generation unit level heat rates, which have been broadcast " @@ -53,7 +53,7 @@ "plant_id_eia", "unit_id_pudl", "generator_id", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "fuel_type_code_pudl", "fuel_type_count", "prime_mover_code", @@ -71,7 +71,7 @@ for freq in AGG_FREQS } | { - f"capacity_factor_by_generator_{freq}": { + f"_out_eia__{freq}_capacity_factor_by_generator": { "description": ( f"{freq.title()} estimates of generator capacity factor. Capacity " "factor is calculated based on reported generator capacity and the " @@ -100,7 +100,7 @@ for freq in AGG_FREQS } | { - f"fuel_cost_by_generator_{freq}": { + f"_out_eia__{freq}_fuel_cost_by_generator": { "description": ( f"{freq.title()} estimate of per-generator fuel costs both per MMBTU " "and per MWh. These calculations are based on the allocation of net " @@ -126,7 +126,7 @@ "fuel_type_code_pudl", "fuel_cost_from_eiaapi", "fuel_cost_per_mmbtu", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "fuel_cost_per_mwh", ], "primary_key": [ @@ -142,16 +142,16 @@ for freq in AGG_FREQS } | { - f"mcoe_{freq}": { + f"_out_eia__{freq}_derived_generator_attributes": { "description": ( f"{freq.title()} generator capacity factor, heat rate, fuel cost per MMBTU and fuel cost " "per MWh. These calculations are based on the allocation of net generation reported on " "the basis of plant, prime mover and energy source to individual generators. Heat rates " "by generator-month are estimated by using allocated estimates for per-generator net " - "generation and fuel consumption as well as the :ref:`boiler_fuel_eia923` table, which " + "generation and fuel consumption as well as the :ref:`core_eia923__monthly_boiler_fuel` table, which " "reports fuel consumed by boiler. Heat rates are necessary to estimate the amount of fuel " "consumed by a generation unit, and thus the fuel cost per MWh generated. Plant specific " - "fuel prices are taken from the :ref:`fuel_receipts_costs_eia923` table, which only has " + "fuel prices are taken from the :ref:`core_eia923__monthly_fuel_receipts_costs` table, which only has " "~70% coverage, leading to some generators with heat rate estimates still lacking fuel " "cost estimates." ), @@ -165,7 +165,7 @@ "fuel_cost_from_eiaapi", "fuel_cost_per_mmbtu", "fuel_cost_per_mwh", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "net_generation_mwh", "total_fuel_cost", "total_mmbtu", @@ -183,12 +183,10 @@ for freq in AGG_FREQS } | { - f"mcoe_generators_{freq}": { + f"out_eia__{freq}_generators": { "description": ( - f"The marginal cost of energy table with {freq.title()} generator capacity factor, " - "heat rate, fuel cost per MMBTU and fuel cost, but with additional potentially " - "useful generator attributes merged on." - "See the description in the MCOE table for more explanation of these calculations." + f"{freq.title()} all generator attributes including calculated capacity factor, " + "heat rate, fuel cost per MMBTU and fuel cost." ), "schema": { "fields": [ @@ -214,10 +212,94 @@ "fuel_cost_from_eiaapi", "fuel_cost_per_mmbtu", "fuel_cost_per_mwh", - "heat_rate_mmbtu_mwh", + "unit_heat_rate_mmbtu_per_mwh", "net_generation_mwh", "total_fuel_cost", "total_mmbtu", + "associated_combined_heat_power", + "bga_source", + "bypass_heat_recovery", + "carbon_capture", + "city", + "cofire_fuels", + "county", + "current_planned_generator_operating_date", + "data_maturity", + "deliver_power_transgrid", + "distributed_generation", + "duct_burners", + "energy_source_1_transport_1", + "energy_source_1_transport_2", + "energy_source_1_transport_3", + "energy_source_2_transport_1", + "energy_source_2_transport_2", + "energy_source_2_transport_3", + "energy_source_code_2", + "energy_source_code_3", + "energy_source_code_4", + "energy_source_code_5", + "energy_source_code_6", + "energy_storage_capacity_mwh", + "ferc_qualifying_facility", + "fluidized_bed_tech", + "fuel_type_count", + "latitude", + "longitude", + "minimum_load_mw", + "multiple_fuels", + "nameplate_power_factor", + "net_capacity_mwdc", + "operating_switch", + "operational_status_code", + "original_planned_generator_operating_date", + "other_combustion_tech", + "other_modifications_date", + "other_planned_modifications", + "owned_by_non_utility", + "ownership_code", + "planned_derate_date", + "planned_energy_source_code_1", + "planned_modifications", + "planned_net_summer_capacity_derate_mw", + "planned_net_summer_capacity_uprate_mw", + "planned_net_winter_capacity_derate_mw", + "planned_net_winter_capacity_uprate_mw", + "planned_new_capacity_mw", + "planned_new_prime_mover_code", + "planned_repower_date", + "planned_uprate_date", + "previously_canceled", + "pulverized_coal_tech", + "reactive_power_output_mvar", + "rto_iso_lmp_node_id", + "rto_iso_location_wholesale_reporting_id", + "solid_fuel_gasification", + "startup_source_code_1", + "startup_source_code_2", + "startup_source_code_3", + "startup_source_code_4", + "state", + "stoker_tech", + "street_address", + "subcritical_tech", + "summer_capacity_estimate", + "summer_capacity_mw", + "summer_estimated_capability_mw", + "supercritical_tech", + "switch_oil_gas", + "syncronized_transmission_grid", + "time_cold_shutdown_full_load_code", + "timezone", + "topping_bottoming_code", + "turbines_inverters_hydrokinetics", + "turbines_num", + "ultrasupercritical_tech", + "uprate_derate_completed_date", + "uprate_derate_during_year", + "winter_capacity_estimate", + "winter_capacity_mw", + "winter_estimated_capability_mw", + "zip_code", ], "primary_key": [ "report_date", diff --git a/src/pudl/metadata/resources/pudl.py b/src/pudl/metadata/resources/pudl.py index fec1782ab2..a58f20f786 100644 --- a/src/pudl/metadata/resources/pudl.py +++ b/src/pudl/metadata/resources/pudl.py @@ -5,7 +5,7 @@ from typing import Any RESOURCE_METADATA: dict[str, dict[str, Any]] = { - "political_subdivisions": { + "core_pudl__codes_subdivisions": { "title": "Political Subdivisions", "description": ( "Various static attributes associated with states, provinces, and other " @@ -31,7 +31,7 @@ "field_namespace": "pudl", "sources": ["pudl"], }, - "plants_pudl": { + "core_pudl__entity_plants_pudl": { "title": "PUDL Plants", "description": "Home table for PUDL assigned plant IDs. These IDs are manually generated each year when new FERC and EIA reporting is integrated, and any newly identified plants are added to the list with a new ID. Each ID maps to a power plant which is reported in at least one FERC or EIA data set. This table is read in from a spreadsheet stored in the PUDL repository: src/pudl/package_data/glue/pudl_id_mapping.xlsx", "schema": { @@ -43,7 +43,7 @@ "field_namespace": "pudl", "sources": ["pudl"], }, - "utilities_pudl": { + "core_pudl__entity_utilities_pudl": { "title": "PUDL Utilities", "description": "Home table for PUDL assigned utility IDs. These IDs are manually generated each year when new FERC and EIA reporting is integrated, and any newly found utilities are added to the list with a new ID. Each ID maps to a power plant owning or operating entity which is reported in at least one FERC or EIA data set. This table is read in from a spreadsheet stored in the PUDL repository: src/pudl/package_data/glue/pudl_id_mapping.xlsx", "schema": { @@ -55,7 +55,7 @@ "field_namespace": "pudl", "sources": ["pudl"], }, - "utility_plant_assn": { + "core_pudl__assn_utilities_plants": { "schema": { "fields": ["utility_id_pudl", "plant_id_pudl"], "primary_key": ["utility_id_pudl", "plant_id_pudl"], @@ -64,7 +64,7 @@ "field_namespace": "pudl", "sources": ["pudl"], }, - "datasources": { + "core_pudl__codes_datasources": { "schema": { "fields": [ "datasource", @@ -78,7 +78,7 @@ "field_namespace": "pudl", "sources": ["pudl"], }, - "predicted_state_hourly_demand": { + "out_ferc714__hourly_predicted_state_demand": { "schema": { "fields": [ "state_id_fips", diff --git a/src/pudl/metadata/sources.py b/src/pudl/metadata/sources.py index 11f8fcc1d6..37105ebc0d 100644 --- a/src/pudl/metadata/sources.py +++ b/src/pudl/metadata/sources.py @@ -299,7 +299,7 @@ "license_raw": LICENSES["us-govt"], "license_pudl": LICENSES["cc-by-4.0"], }, - "epacamd_eia": { + "core_epa__assn_eia_epacamd": { "title": "EPA CAMD to EIA Power Sector Data Crosswalk", "path": "https://github.com/USEPA/camd-eia-crosswalk", "description": ( diff --git a/src/pudl/metadata/templates/datasette-metadata.yml.jinja b/src/pudl/metadata/templates/datasette-metadata.yml.jinja index 38466bb359..464800755d 100644 --- a/src/pudl/metadata/templates/datasette-metadata.yml.jinja +++ b/src/pudl/metadata/templates/datasette-metadata.yml.jinja @@ -42,12 +42,15 @@ databases: Catalyst Cooperative as part of the Public Utility Data Liberation Project.

-

Caution:

+

Note:

    -
  • Please note that tables beginning with "denorm_" are temporary tables whose - names and metadata will shortly change, as we migrate new tables into our database.
    • -
  • The structure of the data and the API are not necessarily stable, so don't - build any critical infrastructure on top of this just yet.
    • +
  • We recommend working + with tables with the ``out_`` prefix as these tables contain the most complete + data. +
    • +
  • For more information about the different types of tables, read through + PUDL's naming conventions +
  • If you find something wrong, please make an issue on GitHub to let us know.
    • diff --git a/src/pudl/output/censusdp1tract.py b/src/pudl/output/censusdp1tract.py index 85fb6a0793..8f1eeccd29 100644 --- a/src/pudl/output/censusdp1tract.py +++ b/src/pudl/output/censusdp1tract.py @@ -15,7 +15,7 @@ def census_asset_factory(layer: str) -> AssetsDefinition: @asset( ins={"censusdp1tract_to_sqlite": AssetIn("censusdp1tract_to_sqlite")}, - name=f"{layer}_censusdp1", + name=f"core_censusdp1__entity_{layer}", ) def census_layer(censusdp1tract_to_sqlite, **kwargs) -> gpd.GeoDataFrame: """Select one layer from the Census DP1 database. diff --git a/src/pudl/output/eia.py b/src/pudl/output/eia.py index 38c696cc3c..29a12270da 100644 --- a/src/pudl/output/eia.py +++ b/src/pudl/output/eia.py @@ -12,26 +12,33 @@ @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_utilities_eia( - utilities_entity_eia: pd.DataFrame, - utilities_eia860: pd.DataFrame, - utilities_eia: pd.DataFrame, +def out_eia__yearly_utilities( + core_eia__entity_utilities: pd.DataFrame, + core_eia860__scd_utilities: pd.DataFrame, + core_pudl__assn_eia_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull all fields from the EIA Utilities table. Args: - utilities_entity_eia: EIA utility entity table. - utilities_eia860: EIA 860 annual utility table. - utilities_eia: Associations between EIA utilities and pudl utility IDs. + core_eia__entity_utilities: EIA utility entity table. + core_eia860__scd_utilities: EIA 860 annual utility table. + core_pudl__assn_eia_pudl_utilities: Associations between EIA utilities and pudl utility IDs. Returns: A DataFrame containing utility attributes from EIA Forms 860 and 923. """ - utilities_eia = utilities_eia[["utility_id_eia", "utility_id_pudl"]] + core_pudl__assn_eia_pudl_utilities = core_pudl__assn_eia_pudl_utilities[ + ["utility_id_eia", "utility_id_pudl"] + ] + out_df = pd.merge( + core_eia__entity_utilities, + core_eia860__scd_utilities, + how="left", + on=["utility_id_eia"], + ) out_df = pd.merge( - utilities_entity_eia, utilities_eia860, how="left", on=["utility_id_eia"] + out_df, core_pudl__assn_eia_pudl_utilities, how="left", on=["utility_id_eia"] ) - out_df = pd.merge(out_df, utilities_eia, how="left", on=["utility_id_eia"]) out_df = out_df.assign(report_date=lambda x: pd.to_datetime(x.report_date)).dropna( subset=["report_date", "utility_id_eia"] ) @@ -46,33 +53,40 @@ def denorm_utilities_eia( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_eia( - plants_entity_eia: pd.DataFrame, - plants_eia860: pd.DataFrame, - plants_eia: pd.DataFrame, - utilities_eia: pd.DataFrame, +def out_eia__yearly_plants( + core_eia__entity_plants: pd.DataFrame, + core_eia860__scd_plants: pd.DataFrame, + core_pudl__assn_eia_pudl_plants: pd.DataFrame, + core_pudl__assn_eia_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull all fields from the EIA Plants tables. Args: - plants_entity_eia: EIA plant entity table. - plants_eia860: EIA 860 annual plant attribute table. - plants_eia: Associations between EIA plants and pudl utility IDs. - utilities_eia: EIA utility ID table. + core_eia__entity_plants: EIA plant entity table. + core_eia860__scd_plants: EIA 860 annual plant attribute table. + core_pudl__assn_eia_pudl_plants: Associations between EIA plants and pudl utility IDs. + core_pudl__assn_eia_pudl_utilities: EIA utility ID table. Returns: A DataFrame containing plant attributes from EIA Forms 860 and 923 """ - plants_eia860 = plants_eia860.assign( + core_eia860__scd_plants = core_eia860__scd_plants.assign( report_date=lambda x: pd.to_datetime(x.report_date) ) - plants_eia = plants_eia[["plant_id_eia", "plant_id_pudl"]] + core_pudl__assn_eia_pudl_plants = core_pudl__assn_eia_pudl_plants[ + ["plant_id_eia", "plant_id_pudl"] + ] out_df = ( - pd.merge(plants_entity_eia, plants_eia860, how="left", on=["plant_id_eia"]) - .merge(plants_eia, how="left", on=["plant_id_eia"]) - .merge(utilities_eia, how="left", on=["utility_id_eia"]) + pd.merge( + core_eia__entity_plants, + core_eia860__scd_plants, + how="left", + on=["plant_id_eia"], + ) + .merge(core_pudl__assn_eia_pudl_plants, how="left", on=["plant_id_eia"]) + .merge(core_pudl__assn_eia_pudl_utilities, how="left", on=["utility_id_eia"]) .dropna(subset=["report_date", "plant_id_eia"]) .pipe(fill_in_missing_ba_codes) ) @@ -100,23 +114,23 @@ def denorm_plants_eia( }, compute_kind="Python", ) -def denorm_generators_eia( +def _out_eia__yearly_generators( context, - generators_eia860: pd.DataFrame, - generators_entity_eia: pd.DataFrame, - plants_entity_eia: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, - boiler_generator_assn_eia860: pd.DataFrame, + core_eia860__scd_generators: pd.DataFrame, + core_eia__entity_generators: pd.DataFrame, + core_eia__entity_plants: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, + core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: """Pull all fields from the EIA Utilities table. Args: context: A Dagster context object. - generators_eia860: EIA 860 annual generator table. - generators_entity_eia: EIA generators entity table. - plants_entity_eia: EIA plant entity table. - denorm_plants_utilities_eia: Denormalized plant_utility EIA ID table. - boiler_generator_assn_eia860: Associations between EIA boiler and generator IDs. + core_eia860__scd_generators: EIA 860 annual generator table. + core_eia__entity_generators: EIA generators entity table. + core_eia__entity_plants: EIA plant entity table. + _out_eia__plants_utilities: Denormalized plant_utility EIA ID table. + core_eia860__assn_boiler_generator: Associations between EIA boiler and generator IDs. Returns: A DataFrame containing all the fields of the EIA 860 Utilities table. @@ -124,11 +138,14 @@ def denorm_generators_eia( # Almost all the info we need will come from here. out_df = pd.merge( - generators_eia860, plants_entity_eia, how="left", on=["plant_id_eia"] + core_eia860__scd_generators, + core_eia__entity_plants, + how="left", + on=["plant_id_eia"], ) out_df = pd.merge( out_df, - generators_entity_eia, + core_eia__entity_generators, how="left", on=["plant_id_eia", "generator_id"], ) @@ -136,7 +153,7 @@ def denorm_generators_eia( out_df.report_date = pd.to_datetime(out_df.report_date) # Bring in some generic plant & utility information: - pu_eia = denorm_plants_utilities_eia.drop( + pu_eia = _out_eia__plants_utilities.drop( ["plant_name_eia", "utility_id_eia", "data_maturity"], axis="columns" ) out_df = pd.merge(out_df, pu_eia, on=["report_date", "plant_id_eia"], how="left") @@ -145,7 +162,7 @@ def denorm_generators_eia( # Pull the BGA table and make it unit-generator only: out_df = pd.merge( out_df, - boiler_generator_assn_eia860[ + core_eia860__assn_boiler_generator[ [ "report_date", "plant_id_eia", @@ -205,12 +222,12 @@ def denorm_generators_eia( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_boilers_eia( - boilers_eia860: pd.DataFrame, - boilers_entity_eia: pd.DataFrame, - plants_entity_eia: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, - boiler_generator_assn_eia860: pd.DataFrame, +def out_eia__yearly_boilers( + core_eia860__scd_boilers: pd.DataFrame, + core_eia__entity_boilers: pd.DataFrame, + core_eia__entity_plants: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, + core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: """Pull all fields reported in the EIA boilers tables. @@ -220,21 +237,24 @@ def denorm_boilers_eia( sources. Arguments: - boilers_eia860: EIA 860 annual boiler table. - boilers_entity_eia: EIA boiler entity table. - plants_entity_eia: EIA plant entity table. - denorm_plants_utilities_eia: Denormalized plant_utility EIA ID table. - boiler_generator_assn_eia860: Associations between EIA boiler and generator IDs. + core_eia860__scd_boilers: EIA 860 annual boiler table. + core_eia__entity_boilers: EIA boiler entity table. + core_eia__entity_plants: EIA plant entity table. + _out_eia__plants_utilities: Denormalized plant_utility EIA ID table. + core_eia860__assn_boiler_generator: Associations between EIA boiler and generator IDs. Returns: A DataFrame containing boiler attributes from EIA 860. """ out_df = pd.merge( - boilers_eia860, plants_entity_eia, how="left", on=["plant_id_eia"] + core_eia860__scd_boilers, + core_eia__entity_plants, + how="left", + on=["plant_id_eia"], ) out_df = pd.merge( - out_df, boilers_entity_eia, how="left", on=["plant_id_eia", "boiler_id"] + out_df, core_eia__entity_boilers, how="left", on=["plant_id_eia", "boiler_id"] ) out_df.report_date = pd.to_datetime(out_df.report_date) @@ -242,7 +262,7 @@ def denorm_boilers_eia( # Bring in some generic plant & utility information: out_df = pd.merge( out_df, - denorm_plants_utilities_eia.drop( + _out_eia__plants_utilities.drop( ["plant_name_eia", "data_maturity"], axis="columns" ), on=["report_date", "plant_id_eia"], @@ -253,7 +273,7 @@ def denorm_boilers_eia( # Pull the BGA table and make it unit-boiler only: out_df = pd.merge( out_df, - boiler_generator_assn_eia860[ + core_eia860__assn_boiler_generator[ [ "report_date", "plant_id_eia", @@ -286,9 +306,9 @@ def denorm_boilers_eia( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_utilities_eia( - denorm_plants_eia: pd.DataFrame, - denorm_utilities_eia: pd.DataFrame, +def _out_eia__plants_utilities( + out_eia__yearly_plants: pd.DataFrame, + out_eia__yearly_utilities: pd.DataFrame, ) -> pd.DataFrame: """Create a dataframe of plant and utility IDs and names from EIA 860. @@ -302,14 +322,14 @@ def denorm_plants_utilities_eia( - utility_id_pudl Args: - denorm_plants_eia: Denormalized EIA plants table. - denorm_utilities_eia: Denormalized EIA utilities table. + out_eia__yearly_plants: Denormalized EIA plants table. + out_eia__yearly_utilities: Denormalized EIA utilities table. Returns: A DataFrame containing plant and utility IDs and names from EIA 860. """ # Contains the one-to-one mapping of EIA plants to their operators - plants_eia = denorm_plants_eia.drop( + core_pudl__assn_eia_pudl_plants = out_eia__yearly_plants.drop( [ "utility_id_pudl", "city", @@ -323,8 +343,8 @@ def denorm_plants_utilities_eia( # to avoid duplicate columns on the merge... out_df = pd.merge( - plants_eia, - denorm_utilities_eia.drop(columns=["data_maturity"]), + core_pudl__assn_eia_pudl_plants, + out_eia__yearly_utilities.drop(columns=["data_maturity"]), how="left", on=["report_date", "utility_id_eia"], ) @@ -532,7 +552,7 @@ def fill_generator_technology_description(gens_df: pd.DataFrame) -> pd.DataFrame ``technology_description`` associated with them. Args: - gens_df: A generators_eia860 dataframe containing at least the columns + gens_df: A core_eia860__scd_generators dataframe containing at least the columns ``report_date``, ``plant_id_eia``, ``generator_id``, ``energy_source_code_1``, and ``technology_description``. @@ -645,10 +665,10 @@ def assign_unit_ids(gens_df: pd.DataFrame) -> pd.DataFrame: # units, we give each generator their own unit ID. We do the same for # internal combustion and simple-cycle gas combustion turbines. .pipe(assign_single_gen_unit_ids, prime_mover_codes=["CC", "CS", "GT", "IC"]) - # Nuclear units don't report in boiler_fuel_eia923 or generation_eia923 - # Their fuel consumption is reported as mmbtu in generation_fuel_eia923 - # Their net generation also only shows up in generation_fuel_eia923 - # The generation_fuel_eia923 table records a "nuclear_unit_id" which + # Nuclear units don't report in core_eia923__monthly_boiler_fuel or core_eia923__monthly_generation + # Their fuel consumption is reported as mmbtu in core_eia923__monthly_generation_fuel + # Their net generation also only shows up in core_eia923__monthly_generation_fuel + # The core_eia923__monthly_generation_fuel table records a "nuclear_unit_id" which # appears to be the same as the associated generator_id. However, we # can't use that as a unit_id_pudl since it might have a collision with # other already assigned unit_id_pudl values in the same plant for @@ -664,7 +684,7 @@ def assign_unit_ids(gens_df: pd.DataFrame) -> pd.DataFrame: # In these next 4 assignments, we lump together all steam turbine (ST) # generators that have a consistent simplified fuel_type_code_pudl # across all years within a given plant into the same unit, since we - # won't be able to distinguish them in the generation_fuel_eia923 + # won't be able to distinguish them in the core_eia923__monthly_generation_fuel # table. This will lump together solid fuels like BIT, LIG, SUB, PC etc. # under "coal". There are a few cases in which a generator has truly # changed its fuel type, e.g. coal-to-gas conversions but these are @@ -775,7 +795,7 @@ def fill_unit_ids(gens_df: pd.DataFrame) -> pd.DataFrame: too, which seems like too much deep muddling. Args: - gens_df: An generators_eia860 dataframe, which must + gens_df: An core_eia860__scd_generators dataframe, which must contain columns: report_date, plant_id_eia, generator_id, unit_id_pudl, bga_source. @@ -814,7 +834,7 @@ def max_unit_id_by_plant(gens_df: pd.DataFrame) -> pd.DataFrame: generators and units still available in the dataframe! Args: - gens_df: A generators_eia860 dataframe containing at + gens_df: A core_eia860__scd_generators dataframe containing at least the columns plant_id_eia and unit_id_pudl. Returns: diff --git a/src/pudl/output/eia860.py b/src/pudl/output/eia860.py index 016574117b..32e9606438 100644 --- a/src/pudl/output/eia860.py +++ b/src/pudl/output/eia860.py @@ -7,26 +7,24 @@ @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_ownership_eia860( - denorm_plants_utilities_eia: pd.DataFrame, - utilities_eia: pd.DataFrame, - ownership_eia860: pd.DataFrame, +def out_eia860__yearly_ownership( + _out_eia__plants_utilities: pd.DataFrame, + core_eia860__scd_ownership: pd.DataFrame, + core_pudl__assn_eia_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """A denormalized version of the EIA 860 ownership table. - TODO: Convert to SQL view? - Args: - denorm_plants_utilities_eia: Denormalized table containing plant and utility + _out_eia__plants_utilities: Denormalized table containing plant and utility names and IDs. - utilities_eia: Table of associations between EIA utility IDs and + core_eia860__scd_ownership: EIA 860 ownership table. + core_pudl__assn_eia_pudl_utilities: Table of associations between EIA utility IDs and PUDL Utility IDs. - ownership_eia860: EIA 860 ownership table. Returns: A denormalized version of the EIA 860 ownership table. """ - pu_df = denorm_plants_utilities_eia.loc[ + pu_df = _out_eia__plants_utilities.loc[ :, [ "plant_id_eia", @@ -36,11 +34,16 @@ def denorm_ownership_eia860( ], ] own_df = pd.merge( - ownership_eia860, pu_df, on=["report_date", "plant_id_eia"], how="left" + core_eia860__scd_ownership, + pu_df, + on=["report_date", "plant_id_eia"], + how="left", ).dropna( subset=["report_date", "plant_id_eia", "generator_id", "owner_utility_id_eia"] ) - util_df = utilities_eia.loc[:, ["utility_id_eia", "utility_id_pudl"]] + util_df = core_pudl__assn_eia_pudl_utilities.loc[ + :, ["utility_id_eia", "utility_id_pudl"] + ] own_df = own_df.merge( util_df, how="left", left_on="owner_utility_id_eia", right_on="utility_id_eia" ) @@ -60,21 +63,21 @@ def denorm_ownership_eia860( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_emissions_control_equipment_eia860( - emissions_control_equipment_eia860: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, +def out_eia860__yearly_emissions_control_equipment( + core_eia860__scd_emissions_control_equipment: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """A denormalized version of the EIA 860 emission control equipment table. Args: - emissions_control_equipment_eia860: EIA 860 emissions control equipment table. - denorm_plants_utilities_eia: Denormalized table containing plant and utility + core_eia860__scd_emissions_control_equipment: EIA 860 emissions control equipment table. + _out_eia__plants_utilities: Denormalized table containing plant and utility names and IDs. Returns: A denormalized version of the EIA 860 emissions control equipment table. """ - pu_df = denorm_plants_utilities_eia.loc[ + pu_df = _out_eia__plants_utilities.loc[ :, [ "plant_id_eia", @@ -90,7 +93,7 @@ def denorm_emissions_control_equipment_eia860( columns=["report_date"] ) emce_df = pd.merge( - emissions_control_equipment_eia860, + core_eia860__scd_emissions_control_equipment, pu_df, on=["report_year", "plant_id_eia"], how="left", @@ -99,7 +102,7 @@ def denorm_emissions_control_equipment_eia860( # Add a column for operational status emce_df["operational_status"] = emce_df.operational_status_code.str.upper().map( pudl.helpers.label_map( - CODE_METADATA["operational_status_eia"]["df"], + CODE_METADATA["core_eia__codes_operational_status"]["df"], from_col="code", to_col="operational_status", null_value=pd.NA, diff --git a/src/pudl/output/eia923.py b/src/pudl/output/eia923.py index 2903c3100c..62a9ddf6b8 100644 --- a/src/pudl/output/eia923.py +++ b/src/pudl/output/eia923.py @@ -152,29 +152,29 @@ def drop_ytd_for_annual_tables(df: pd.DataFrame, freq: str) -> pd.DataFrame: # Simple Denormalized Assets ##################################################################################### @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_generation_eia923( - generation_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, - boiler_generator_assn_eia860: pd.DataFrame, +def out_eia923__generation( + core_eia923__monthly_generation: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, + core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: - """Denormalize the :ref:`generation_eia923` table.""" + """Denormalize the :ref:`core_eia923__monthly_generation` table.""" return denorm_by_gen( - generation_eia923, - pu=denorm_plants_utilities_eia, - bga=boiler_generator_assn_eia860, + core_eia923__monthly_generation, + pu=_out_eia__plants_utilities, + bga=core_eia860__assn_boiler_generator, ) @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_generation_fuel_combined_eia923( - generation_fuel_eia923: pd.DataFrame, - generation_fuel_nuclear_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, +def out_eia923__generation_fuel_combined( + core_eia923__monthly_generation_fuel: pd.DataFrame, + core_eia923__monthly_generation_fuel_nuclear: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """Denormalize the `generation_fuel_combined_eia923` table. - This asset first combines the :ref:`generation_fuel_eia923` and - :ref:`generation_fuel_nuclear_eia923` into a single table with a uniform primary + This asset first combines the :ref:`core_eia923__monthly_generation_fuel` and + :ref:`core_eia923__monthly_generation_fuel_nuclear` into a single table with a uniform primary key (consolidating multiple nuclear unit IDs into a single plant record) and then denormalizes it by merging in some addition plant and utility level columns. @@ -201,11 +201,11 @@ def denorm_generation_fuel_combined_eia923( # Rather than enumerating all of the non-data columns, identify them by process of # elimination, in case they change in the future. non_data_cols = list( - set(generation_fuel_nuclear_eia923.columns) + set(core_eia923__monthly_generation_fuel_nuclear.columns) - set(primary_key + sum_cols + other_cols) ) - gfn_gb = generation_fuel_nuclear_eia923.groupby(primary_key) + gfn_gb = core_eia923__monthly_generation_fuel_nuclear.groupby(primary_key) # Ensure that all non-data columns are homogeneous within groups if gfn_gb[non_data_cols].nunique().ne(1).any(axis=None): raise ValueError( @@ -230,34 +230,33 @@ def denorm_generation_fuel_combined_eia923( ) ).reset_index() gf = ( - pd.concat([gfn_agg, generation_fuel_eia923]) + pd.concat([gfn_agg, core_eia923__monthly_generation_fuel]) .sort_values(primary_key) .reset_index(drop=True) ) - return denorm_by_plant(gf, pu=denorm_plants_utilities_eia) + return denorm_by_plant(gf, pu=_out_eia__plants_utilities) @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_boiler_fuel_eia923( - boiler_fuel_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, - boiler_generator_assn_eia860: pd.DataFrame, +def out_eia923__boiler_fuel( + core_eia923__monthly_boiler_fuel: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, + core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: - """Denormalize the :ref:`boiler_fuel_eia923` table. + """Denormalize the :ref:`core_eia923__monthly_boiler_fuel` table. The total heat content is also calculated as it's useful in its own right and required later to calculate average heat content per unit of fuel. """ - boiler_fuel_eia923["fuel_consumed_mmbtu"] = ( - boiler_fuel_eia923["fuel_consumed_units"] - * boiler_fuel_eia923["fuel_mmbtu_per_unit"] + core_eia923__monthly_boiler_fuel["fuel_consumed_mmbtu"] = ( + core_eia923__monthly_boiler_fuel["fuel_consumed_units"] + * core_eia923__monthly_boiler_fuel["fuel_mmbtu_per_unit"] ) - dd = denorm_by_boil( - boiler_fuel_eia923, - pu=denorm_plants_utilities_eia, - bga=boiler_generator_assn_eia860, + return denorm_by_boil( + core_eia923__monthly_boiler_fuel, + pu=_out_eia__plants_utilities, + bga=core_eia860__assn_boiler_generator, ) - return dd @asset( @@ -278,21 +277,23 @@ def denorm_boiler_fuel_eia923( }, compute_kind="Python", ) -def denorm_fuel_receipts_costs_eia923( +def out_eia923__fuel_receipts_costs( context, - fuel_receipts_costs_eia923: pd.DataFrame, - coalmine_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, + core_eia923__monthly_fuel_receipts_costs: pd.DataFrame, + core_eia923__entity_coalmine: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, state_average_fuel_costs_eia: pd.DataFrame, - plants_entity_eia: pd.DataFrame, + core_eia__entity_plants: pd.DataFrame, ) -> pd.DataFrame: - """Denormalize the :ref:`fuel_receipts_costs_eia923` table.""" - coalmine_eia923 = coalmine_eia923.drop(columns=["data_maturity"]) - plant_states = plants_entity_eia[["plant_id_eia", "state"]] + """Denormalize the :ref:`core_eia923__monthly_fuel_receipts_costs` table.""" + core_eia923__entity_coalmine = core_eia923__entity_coalmine.drop( + columns=["data_maturity"] + ) + plant_states = core_eia__entity_plants[["plant_id_eia", "state"]] frc_df = ( pd.merge( - fuel_receipts_costs_eia923, - coalmine_eia923.rename( + core_eia923__monthly_fuel_receipts_costs, + core_eia923__entity_coalmine.rename( columns={ "state": "mine_state", "county_id_fips": "coalmine_county_id_fips", @@ -332,7 +333,7 @@ def denorm_fuel_receipts_costs_eia923( frc_df["total_fuel_cost"] = ( frc_df["fuel_consumed_mmbtu"] * frc_df["fuel_cost_per_mmbtu"] ) - return denorm_by_plant(frc_df, pu=denorm_plants_utilities_eia) + return denorm_by_plant(frc_df, pu=_out_eia__plants_utilities) ##################################################################################### @@ -346,20 +347,20 @@ def time_aggregated_eia923_asset_factory( agg_freqs = {"AS": "yearly", "MS": "monthly"} @asset( - name=f"denorm_generation_{agg_freqs[freq]}_eia923", + name=f"out_eia923__{agg_freqs[freq]}_generation", io_manager_key=io_manager_key, compute_kind="Python", ) def generation_agg_eia923( - denorm_generation_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, - boiler_generator_assn_eia860: pd.DataFrame, + out_eia923__generation: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, + core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: - """Aggregate :ref:`generation_eia923` monthly or annually.""" + """Aggregate :ref:`out_eia923__generation` monthly or annually.""" return ( # Create a date index for grouping based on freq - denorm_generation_eia923.set_index( - pd.DatetimeIndex(denorm_generation_eia923.report_date) + out_eia923__generation.set_index( + pd.DatetimeIndex(out_eia923__generation.report_date) ) .pipe(drop_ytd_for_annual_tables, freq) .groupby( @@ -370,25 +371,25 @@ def generation_agg_eia923( .reset_index() .pipe( denorm_by_gen, - pu=denorm_plants_utilities_eia, - bga=boiler_generator_assn_eia860, + pu=_out_eia__plants_utilities, + bga=core_eia860__assn_boiler_generator, ) ) @asset( - name=f"denorm_generation_fuel_combined_{agg_freqs[freq]}_eia923", + name=f"out_eia923__{agg_freqs[freq]}_generation_fuel_combined", io_manager_key=io_manager_key, compute_kind="Python", ) def generation_fuel_combined_agg_eia923( - denorm_generation_fuel_combined_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, + out_eia923__generation_fuel_combined: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """Aggregate :ref:`generation_fuel_combined_eia923` monthly or annually.""" gf_both = ( # Create a date index for temporal resampling: - denorm_generation_fuel_combined_eia923.set_index( - pd.DatetimeIndex(denorm_generation_fuel_combined_eia923.report_date) + out_eia923__generation_fuel_combined.set_index( + pd.DatetimeIndex(out_eia923__generation_fuel_combined.report_date) ) .pipe(drop_ytd_for_annual_tables, freq) .groupby( @@ -441,29 +442,29 @@ def generation_fuel_combined_agg_eia923( ] ) .reset_index(drop=True) - .pipe(denorm_by_plant, pu=denorm_plants_utilities_eia) + .pipe(denorm_by_plant, pu=_out_eia__plants_utilities) ) @asset( - name=f"denorm_boiler_fuel_{agg_freqs[freq]}_eia923", + name=f"out_eia923__{agg_freqs[freq]}_boiler_fuel", io_manager_key=io_manager_key, compute_kind="Python", ) def boiler_fuel_agg_eia923( - denorm_boiler_fuel_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, - boiler_generator_assn_eia860: pd.DataFrame, + out_eia923__boiler_fuel: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, + core_eia860__assn_boiler_generator: pd.DataFrame, ) -> pd.DataFrame: - """Aggregate :ref:`boiler_fuel_eia923` monthly or annually.""" + """Aggregate :ref:`core_eia923__monthly_boiler_fuel` monthly or annually.""" # In order to calculate the weighted average sulfur # content and ash content we need to calculate these totals. return ( - denorm_boiler_fuel_eia923.assign( + out_eia923__boiler_fuel.assign( total_sulfur_content=lambda x: x.fuel_consumed_units * x.sulfur_content_pct, total_ash_content=lambda x: x.fuel_consumed_units * x.ash_content_pct, ) - .set_index(pd.DatetimeIndex(denorm_boiler_fuel_eia923.report_date)) + .set_index(pd.DatetimeIndex(out_eia923__boiler_fuel.report_date)) .pipe(drop_ytd_for_annual_tables, freq) .groupby( by=[ @@ -498,24 +499,24 @@ def boiler_fuel_agg_eia923( .reset_index() .pipe( denorm_by_boil, - pu=denorm_plants_utilities_eia, - bga=boiler_generator_assn_eia860, + pu=_out_eia__plants_utilities, + bga=core_eia860__assn_boiler_generator, ) ) @asset( - name=f"denorm_fuel_receipts_costs_{agg_freqs[freq]}_eia923", + name=f"out_eia923__{agg_freqs[freq]}_fuel_receipts_costs", io_manager_key=io_manager_key, compute_kind="Python", ) def fuel_receipts_costs_agg_eia923( - denorm_fuel_receipts_costs_eia923: pd.DataFrame, - denorm_plants_utilities_eia: pd.DataFrame, + out_eia923__fuel_receipts_costs: pd.DataFrame, + _out_eia__plants_utilities: pd.DataFrame, ) -> pd.DataFrame: - """Aggregate the :ref:`fuel_receipts_costs_eia923` table monthly or annually.""" + """Aggregate the :ref:`core_eia923__monthly_fuel_receipts_costs` table monthly or annually.""" return ( - denorm_fuel_receipts_costs_eia923.set_index( - pd.DatetimeIndex(denorm_fuel_receipts_costs_eia923.report_date) + out_eia923__fuel_receipts_costs.set_index( + pd.DatetimeIndex(out_eia923__fuel_receipts_costs.report_date) ) .assign( total_ash_content=lambda x: x.ash_content_pct * x.fuel_received_units, @@ -572,7 +573,7 @@ def fuel_receipts_costs_agg_eia923( ] ) .reset_index() - .pipe(denorm_by_plant, pu=denorm_plants_utilities_eia) + .pipe(denorm_by_plant, pu=_out_eia__plants_utilities) ) return [ diff --git a/src/pudl/output/eia_bulk_elec.py b/src/pudl/output/eia_bulk_elec.py index 85c61edcc5..21fe311d03 100644 --- a/src/pudl/output/eia_bulk_elec.py +++ b/src/pudl/output/eia_bulk_elec.py @@ -9,15 +9,15 @@ @asset(io_manager_key=None, compute_kind="Python") def state_average_fuel_costs_eia( - fuel_receipts_costs_aggs_eia: pd.DataFrame, + core_eia__yearly_fuel_receipts_costs_aggs: pd.DataFrame, ) -> pd.DataFrame: """Get state-level average fuel costs from EIA's bulk electricity data. This data is used to fill in missing fuel prices in the - :ref:`fuel_receipts_costs_eia923` table. It was created as a drop-in replacement + :ref:`core_eia923__monthly_fuel_receipts_costs` table. It was created as a drop-in replacement for data we were previously obtaining from EIA's unreliable API. """ - aggregates = fuel_receipts_costs_aggs_eia.loc[ + aggregates = core_eia__yearly_fuel_receipts_costs_aggs.loc[ :, [ "report_date", diff --git a/src/pudl/output/ferc1.py b/src/pudl/output/ferc1.py index 3594701306..56d2b14881 100644 --- a/src/pudl/output/ferc1.py +++ b/src/pudl/output/ferc1.py @@ -31,7 +31,7 @@ EXPLOSION_CALCULATION_TOLERANCES: dict[str, GroupMetricChecks] = { - "income_statement_ferc1": GroupMetricChecks( + "core_ferc1__yearly_income_statements_sched114": GroupMetricChecks( groups_to_check=[ "ungrouped", "report_year", @@ -61,7 +61,7 @@ ), ), ), - "balance_sheet_assets_ferc1": GroupMetricChecks( + "core_ferc1__yearly_balance_sheet_assets_sched110": GroupMetricChecks( groups_to_check=[ "ungrouped", "report_year", @@ -91,7 +91,7 @@ ), ), ), - "balance_sheet_liabilities_ferc1": GroupMetricChecks( + "core_ferc1__yearly_balance_sheet_liabilities_sched110": GroupMetricChecks( groups_to_check=[ "ungrouped", "report_year", @@ -190,17 +190,22 @@ @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_utilities_ferc1( - plants_ferc1: pd.DataFrame, - utilities_ferc1: pd.DataFrame, +def _out_ferc1__yearly_plants_utilities( + core_pudl__assn_ferc1_pudl_plants: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """A denormalized table containing FERC plant and utility names and IDs.""" - return pd.merge(plants_ferc1, utilities_ferc1, on="utility_id_ferc1") + return pd.merge( + core_pudl__assn_ferc1_pudl_plants, + core_pudl__assn_ferc1_pudl_utilities, + on="utility_id_ferc1", + ) @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_steam_ferc1( - denorm_plants_utilities_ferc1: pd.DataFrame, plants_steam_ferc1: pd.DataFrame +def _out_ferc1__yearly_steam_plants_sched402( + _out_ferc1__yearly_plants_utilities: pd.DataFrame, + core_ferc1__yearly_steam_plants_sched402: pd.DataFrame, ) -> pd.DataFrame: """Select and joins some useful fields from the FERC Form 1 steam table. @@ -211,16 +216,16 @@ def denorm_plants_steam_ferc1( ``capacity_mw``) Args: - denorm_plants_utilities_ferc1: Denormalized dataframe of FERC Form 1 plants and + _out_ferc1__yearly_plants_utilities: Denormalized dataframe of FERC Form 1 plants and utilities data. - plants_steam_ferc1: The normalized FERC Form 1 steam table. + core_ferc1__yearly_steam_plants_sched402: The normalized FERC Form 1 steam table. Returns: A DataFrame containing useful fields from the FERC Form 1 steam table. """ steam_df = ( - plants_steam_ferc1.merge( - denorm_plants_utilities_ferc1, + core_ferc1__yearly_steam_plants_sched402.merge( + _out_ferc1__yearly_plants_utilities, on=["utility_id_ferc1", "plant_name_ferc1"], how="left", ) @@ -253,13 +258,14 @@ def denorm_plants_steam_ferc1( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_small_ferc1( - plants_small_ferc1: pd.DataFrame, denorm_plants_utilities_ferc1: pd.DataFrame +def _out_ferc1__yearly_small_plants_sched410( + core_ferc1__yearly_small_plants_sched410: pd.DataFrame, + _out_ferc1__yearly_plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe related to the FERC Form 1 small plants.""" plants_small_df = ( - plants_small_ferc1.merge( - denorm_plants_utilities_ferc1, + core_ferc1__yearly_small_plants_sched410.merge( + _out_ferc1__yearly_plants_utilities, on=["utility_id_ferc1", "plant_name_ferc1"], how="left", ) @@ -289,13 +295,14 @@ def denorm_plants_small_ferc1( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_hydro_ferc1( - plants_hydro_ferc1: pd.DataFrame, denorm_plants_utilities_ferc1: pd.DataFrame +def _out_ferc1__yearly_hydroelectric_plants_sched406( + core_ferc1__yearly_hydroelectric_plants_sched406: pd.DataFrame, + _out_ferc1__yearly_plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe related to the FERC Form 1 hydro plants.""" plants_hydro_df = ( - plants_hydro_ferc1.merge( - denorm_plants_utilities_ferc1, + core_ferc1__yearly_hydroelectric_plants_sched406.merge( + _out_ferc1__yearly_plants_utilities, on=["utility_id_ferc1", "plant_name_ferc1"], how="left", ) @@ -319,14 +326,14 @@ def denorm_plants_hydro_ferc1( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_pumped_storage_ferc1( - plants_pumped_storage_ferc1: pd.DataFrame, - denorm_plants_utilities_ferc1: pd.DataFrame, +def _out_ferc1__yearly_pumped_storage_plants_sched408( + core_ferc1__yearly_pumped_storage_plants_sched408: pd.DataFrame, + _out_ferc1__yearly_plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a dataframe of FERC Form 1 Pumped Storage plant data.""" pumped_storage_df = ( - plants_pumped_storage_ferc1.merge( - denorm_plants_utilities_ferc1, + core_ferc1__yearly_pumped_storage_plants_sched408.merge( + _out_ferc1__yearly_plants_utilities, on=["utility_id_ferc1", "plant_name_ferc1"], how="left", ) @@ -350,8 +357,9 @@ def denorm_plants_pumped_storage_ferc1( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_fuel_ferc1( - fuel_ferc1: pd.DataFrame, denorm_plants_utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_steam_plants_fuel_sched402( + core_ferc1__yearly_steam_plants_fuel_sched402: pd.DataFrame, + _out_ferc1__yearly_plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe related to FERC Form 1 fuel information. @@ -371,14 +379,14 @@ def denorm_fuel_ferc1( information. """ fuel_df = ( - fuel_ferc1.assign( + core_ferc1__yearly_steam_plants_fuel_sched402.assign( fuel_consumed_mmbtu=lambda x: x["fuel_consumed_units"] * x["fuel_mmbtu_per_unit"], fuel_consumed_total_cost=lambda x: x["fuel_consumed_units"] * x["fuel_cost_per_unit_burned"], ) .merge( - denorm_plants_utilities_ferc1, + _out_ferc1__yearly_plants_utilities, on=["utility_id_ferc1", "plant_name_ferc1"], ) .pipe( @@ -397,32 +405,38 @@ def denorm_fuel_ferc1( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_purchased_power_ferc1( - purchased_power_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_purchased_power_and_exchanges_sched326( + core_ferc1__yearly_purchased_power_and_exchanges_sched326: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - purchased_power_df = purchased_power_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "seller_name", - "record_id", - ], + purchased_power_df = ( + core_ferc1__yearly_purchased_power_and_exchanges_sched326.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "seller_name", + "record_id", + ], + ) ) return purchased_power_df @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plant_in_service_ferc1( - plant_in_service_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_plant_in_service_sched204( + core_ferc1__yearly_plant_in_service_sched204: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a dataframe of FERC Form 1 Electric Plant in Service data.""" - pis_df = plant_in_service_ferc1.merge(utilities_ferc1, on="utility_id_ferc1").pipe( + pis_df = core_ferc1__yearly_plant_in_service_sched204.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( pudl.helpers.organize_cols, [ "report_year", @@ -436,77 +450,86 @@ def denorm_plant_in_service_ferc1( @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_balance_sheet_assets_ferc1( - balance_sheet_assets_ferc1: pd.DataFrame, - utilities_ferc1: pd.DataFrame, +def out_ferc1__yearly_balance_sheet_assets_sched110( + core_ferc1__yearly_balance_sheet_assets_sched110: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 balance sheet assets data.""" - denorm_balance_sheet_assets_ferc1 = balance_sheet_assets_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "asset_type", - ], + out_ferc1__yearly_balance_sheet_assets_sched110 = ( + core_ferc1__yearly_balance_sheet_assets_sched110.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "asset_type", + ], + ) ) - return denorm_balance_sheet_assets_ferc1 + return out_ferc1__yearly_balance_sheet_assets_sched110 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_balance_sheet_liabilities_ferc1( - balance_sheet_liabilities_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_balance_sheet_liabilities_sched110( + core_ferc1__yearly_balance_sheet_liabilities_sched110: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 balance_sheet liabilities data.""" - denorm_balance_sheet_liabilities_ferc1 = balance_sheet_liabilities_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "liability_type", - ], + out_ferc1__yearly_balance_sheet_liabilities_sched110 = ( + core_ferc1__yearly_balance_sheet_liabilities_sched110.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "liability_type", + ], + ) ) - return denorm_balance_sheet_liabilities_ferc1 + return out_ferc1__yearly_balance_sheet_liabilities_sched110 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_cash_flow_ferc1( - cash_flow_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_cash_flows_sched120( + core_ferc1__yearly_cash_flows_sched120: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 cash flow data.""" - denorm_cash_flow_ferc1 = cash_flow_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "amount_type", - ], + out_ferc1__yearly_cash_flows_sched120 = ( + core_ferc1__yearly_cash_flows_sched120.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "amount_type", + ], + ) ) - return denorm_cash_flow_ferc1 + return out_ferc1__yearly_cash_flows_sched120 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_depreciation_amortization_summary_ferc1( - depreciation_amortization_summary_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_depreciation_summary_sched336( + core_ferc1__yearly_depreciation_summary_sched336: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 depreciation amortization data.""" - denorm_depreciation_amortization_summary_ferc1 = ( - depreciation_amortization_summary_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" + out_ferc1__yearly_depreciation_summary_sched336 = ( + core_ferc1__yearly_depreciation_summary_sched336.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" ).pipe( pudl.helpers.organize_cols, [ @@ -520,17 +543,18 @@ def denorm_depreciation_amortization_summary_ferc1( ], ) ) - return denorm_depreciation_amortization_summary_ferc1 + return out_ferc1__yearly_depreciation_summary_sched336 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_electric_energy_dispositions_ferc1( - electric_energy_dispositions_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_energy_dispositions_sched401( + core_ferc1__yearly_energy_dispositions_sched401: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 energy dispositions data.""" - denorm_electric_energy_dispositions_ferc1 = ( - electric_energy_dispositions_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" + out_ferc1__yearly_energy_dispositions_sched401 = ( + core_ferc1__yearly_energy_dispositions_sched401.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" ).pipe( pudl.helpers.organize_cols, [ @@ -543,81 +567,90 @@ def denorm_electric_energy_dispositions_ferc1( ], ) ) - return denorm_electric_energy_dispositions_ferc1 + return out_ferc1__yearly_energy_dispositions_sched401 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_electric_energy_sources_ferc1( - electric_energy_sources_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_energy_sources_sched401( + core_ferc1__yearly_energy_sources_sched401: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_electric_energy_sources_ferc1 = electric_energy_sources_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "energy_source_type", - ], + out_ferc1__yearly_energy_sources_sched401 = ( + core_ferc1__yearly_energy_sources_sched401.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "energy_source_type", + ], + ) ) - return denorm_electric_energy_sources_ferc1 + return out_ferc1__yearly_energy_sources_sched401 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_electric_operating_expenses_ferc1( - electric_operating_expenses_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_operating_expenses_sched320( + core_ferc1__yearly_operating_expenses_sched320: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_electric_operating_expenses_ferc1 = electric_operating_expenses_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "expense_type", - ], + out_ferc1__yearly_operating_expenses_sched320 = ( + core_ferc1__yearly_operating_expenses_sched320.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "expense_type", + ], + ) ) - return denorm_electric_operating_expenses_ferc1 + return out_ferc1__yearly_operating_expenses_sched320 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_electric_operating_revenues_ferc1( - electric_operating_revenues_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_operating_revenues_sched300( + core_ferc1__yearly_operating_revenues_sched300: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_electric_operating_revenues_ferc1 = electric_operating_revenues_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "revenue_type", - ], + out_ferc1__yearly_operating_revenues_sched300 = ( + core_ferc1__yearly_operating_revenues_sched300.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "revenue_type", + ], + ) ) - return denorm_electric_operating_revenues_ferc1 + return out_ferc1__yearly_operating_revenues_sched300 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_electric_plant_depreciation_changes_ferc1( - electric_plant_depreciation_changes_ferc1: pd.DataFrame, - utilities_ferc1: pd.DataFrame, +def out_ferc1__yearly_depreciation_changes_sched219( + core_ferc1__yearly_depreciation_changes_sched219: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_electric_plant_depreciation_changes_ferc1 = ( - electric_plant_depreciation_changes_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" + out_ferc1__yearly_depreciation_changes_sched219 = ( + core_ferc1__yearly_depreciation_changes_sched219.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" ).pipe( pudl.helpers.organize_cols, [ @@ -632,18 +665,18 @@ def denorm_electric_plant_depreciation_changes_ferc1( ], ) ) - return denorm_electric_plant_depreciation_changes_ferc1 + return out_ferc1__yearly_depreciation_changes_sched219 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_electric_plant_depreciation_functional_ferc1( - electric_plant_depreciation_functional_ferc1: pd.DataFrame, - utilities_ferc1: pd.DataFrame, +def out_ferc1__yearly_depreciation_by_function_sched219( + core_ferc1__yearly_depreciation_by_function_sched219: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_electric_plant_depreciation_functional_ferc1 = ( - electric_plant_depreciation_functional_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" + out_ferc1__yearly_depreciation_by_function_sched219 = ( + core_ferc1__yearly_depreciation_by_function_sched219.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" ).pipe( pudl.helpers.organize_cols, [ @@ -658,18 +691,18 @@ def denorm_electric_plant_depreciation_functional_ferc1( ], ) ) - return denorm_electric_plant_depreciation_functional_ferc1 + return out_ferc1__yearly_depreciation_by_function_sched219 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_electricity_sales_by_rate_schedule_ferc1( - electricity_sales_by_rate_schedule_ferc1: pd.DataFrame, - utilities_ferc1: pd.DataFrame, +def out_ferc1__yearly_sales_by_rate_schedules_sched304( + core_ferc1__yearly_sales_by_rate_schedules_sched304: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_electricity_sales_by_rate_schedule_ferc1 = ( - electricity_sales_by_rate_schedule_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" + out_ferc1__yearly_sales_by_rate_schedules_sched304 = ( + core_ferc1__yearly_sales_by_rate_schedules_sched304.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" ).pipe( pudl.helpers.organize_cols, [ @@ -681,39 +714,43 @@ def denorm_electricity_sales_by_rate_schedule_ferc1( ], ) ) - return denorm_electricity_sales_by_rate_schedule_ferc1 + return out_ferc1__yearly_sales_by_rate_schedules_sched304 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_income_statement_ferc1( - income_statement_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_income_statements_sched114( + core_ferc1__yearly_income_statements_sched114: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_income_statement_ferc1 = income_statement_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "utility_type", - "income_type", - ], + out_ferc1__yearly_income_statements_sched114 = ( + core_ferc1__yearly_income_statements_sched114.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "utility_type", + "income_type", + ], + ) ) - return denorm_income_statement_ferc1 + return out_ferc1__yearly_income_statements_sched114 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_other_regulatory_liabilities_ferc1( - other_regulatory_liabilities_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_other_regulatory_liabilities_sched278( + core_ferc1__yearly_other_regulatory_liabilities_sched278: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_other_regulatory_liabilities_ferc1 = ( - other_regulatory_liabilities_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" + out_ferc1__yearly_other_regulatory_liabilities_sched278 = ( + core_ferc1__yearly_other_regulatory_liabilities_sched278.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" ).pipe( pudl.helpers.organize_cols, [ @@ -724,77 +761,86 @@ def denorm_other_regulatory_liabilities_ferc1( ], ) ) - return denorm_other_regulatory_liabilities_ferc1 + return out_ferc1__yearly_other_regulatory_liabilities_sched278 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_retained_earnings_ferc1( - retained_earnings_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_retained_earnings_sched118( + core_ferc1__yearly_retained_earnings_sched118: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_retained_earnings_ferc1 = retained_earnings_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "earnings_type", - ], + out_ferc1__yearly_retained_earnings_sched118 = ( + core_ferc1__yearly_retained_earnings_sched118.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "earnings_type", + ], + ) ) - return denorm_retained_earnings_ferc1 + return out_ferc1__yearly_retained_earnings_sched118 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_transmission_statistics_ferc1( - transmission_statistics_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_transmission_lines_sched422( + core_ferc1__yearly_transmission_lines_sched422: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_transmission_statistics_ferc1 = transmission_statistics_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - ], + out_ferc1__yearly_transmission_lines_sched422 = ( + core_ferc1__yearly_transmission_lines_sched422.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + ], + ) ) - return denorm_transmission_statistics_ferc1 + return out_ferc1__yearly_transmission_lines_sched422 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_utility_plant_summary_ferc1( - utility_plant_summary_ferc1: pd.DataFrame, utilities_ferc1: pd.DataFrame +def out_ferc1__yearly_utility_plant_summary_sched200( + core_ferc1__yearly_utility_plant_summary_sched200: pd.DataFrame, + core_pudl__assn_ferc1_pudl_utilities: pd.DataFrame, ) -> pd.DataFrame: """Pull a useful dataframe of FERC Form 1 Purchased Power data.""" - denorm_utility_plant_summary_ferc1 = utility_plant_summary_ferc1.merge( - utilities_ferc1, on="utility_id_ferc1" - ).pipe( - pudl.helpers.organize_cols, - [ - "report_year", - "utility_id_ferc1", - "utility_id_pudl", - "utility_name_ferc1", - "record_id", - "utility_type", - "utility_plant_asset_type", - ], + out_ferc1__yearly_utility_plant_summary_sched200 = ( + core_ferc1__yearly_utility_plant_summary_sched200.merge( + core_pudl__assn_ferc1_pudl_utilities, on="utility_id_ferc1" + ).pipe( + pudl.helpers.organize_cols, + [ + "report_year", + "utility_id_ferc1", + "utility_id_pudl", + "utility_name_ferc1", + "record_id", + "utility_type", + "utility_plant_asset_type", + ], + ) ) - return denorm_utility_plant_summary_ferc1 + return out_ferc1__yearly_utility_plant_summary_sched200 @asset(io_manager_key="pudl_sqlite_io_manager", compute_kind="Python") -def denorm_plants_all_ferc1( - denorm_plants_steam_ferc1: pd.DataFrame, - denorm_plants_small_ferc1: pd.DataFrame, - denorm_plants_hydro_ferc1: pd.DataFrame, - denorm_plants_pumped_storage_ferc1: pd.DataFrame, +def out_ferc1__yearly_all_plants( + _out_ferc1__yearly_steam_plants_sched402: pd.DataFrame, + core_ferc1__yearly_small_plants_sched410: pd.DataFrame, + _out_ferc1__yearly_hydroelectric_plants_sched406: pd.DataFrame, + _out_ferc1__yearly_pumped_storage_plants_sched408: pd.DataFrame, ) -> pd.DataFrame: """Combine the steam, small generators, hydro, and pumped storage tables. @@ -806,21 +852,25 @@ def denorm_plants_all_ferc1( """ # Prep steam table logger.debug("prepping steam table") - steam_df = denorm_plants_steam_ferc1.rename(columns={"opex_plants": "opex_plant"}) + steam_df = _out_ferc1__yearly_steam_plants_sched402.rename( + columns={"opex_plants": "opex_plant"} + ) # Prep hydro tables (Add this to the meta data later) logger.debug("prepping hydro tables") - hydro_df = denorm_plants_hydro_ferc1.rename( + hydro_df = _out_ferc1__yearly_hydroelectric_plants_sched406.rename( columns={"project_num": "ferc_license_id"} ) - pump_df = denorm_plants_pumped_storage_ferc1.rename( + pump_df = _out_ferc1__yearly_pumped_storage_plants_sched408.rename( columns={"project_num": "ferc_license_id"} ) # Combine all the tables together logger.debug("combining all tables") all_df = ( - pd.concat([steam_df, denorm_plants_small_ferc1, hydro_df, pump_df]) + pd.concat( + [steam_df, core_ferc1__yearly_small_plants_sched410, hydro_df, pump_df] + ) .rename( columns={ "fuel_cost": "total_fuel_cost", @@ -850,10 +900,10 @@ def denorm_plants_all_ferc1( }, compute_kind="Python", ) -def denorm_fuel_by_plant_ferc1( +def out_ferc1__yearly_steam_plants_fuel_by_plant_sched402( context, - fuel_ferc1: pd.DataFrame, - denorm_plants_utilities_ferc1: pd.DataFrame, + core_ferc1__yearly_steam_plants_fuel_sched402: pd.DataFrame, + _out_ferc1__yearly_plants_utilities: pd.DataFrame, ) -> pd.DataFrame: """Summarize FERC fuel data by plant for output. @@ -865,8 +915,8 @@ def denorm_fuel_by_plant_ferc1( Args: context: Dagster context object - fuel_ferc1: Normalized FERC fuel table. - denorm_plants_utilities_ferc1: Denormalized table of FERC1 plant & utility IDs. + core_ferc1__yearly_steam_plants_fuel_sched402: Normalized FERC fuel table. + _out_ferc1__yearly_plants_utilities: Denormalized table of FERC1 plant & utility IDs. Returns: A DataFrame with fuel use summarized by plant. @@ -881,15 +931,21 @@ def drop_other_fuel_types(df): return df[df.fuel_type_code_pudl != "other"].copy() thresh = context.op_config["thresh"] + # The existing function expects `fuel_type_code_pudl` to be an object, rather than + # a category. This is a legacy of pre-dagster code, and we convert here to prevent + # further retooling in the code-base. + core_ferc1__yearly_steam_plants_fuel_sched402[ + "fuel_type_code_pudl" + ] = core_ferc1__yearly_steam_plants_fuel_sched402["fuel_type_code_pudl"].astype(str) fuel_categories = list( - pudl.transform.ferc1.FuelFerc1TableTransformer() + pudl.transform.ferc1.SteamPlantsFuelTableTransformer() .params.categorize_strings["fuel_type_code_pudl"] .categories.keys() ) fbp_df = ( - fuel_ferc1.pipe(drop_other_fuel_types) + core_ferc1__yearly_steam_plants_fuel_sched402.pipe(drop_other_fuel_types) .pipe( pudl.analysis.classify_plants_ferc1.fuel_by_plant_ferc1, fuel_categories=fuel_categories, @@ -898,7 +954,8 @@ def drop_other_fuel_types(df): .pipe(pudl.analysis.classify_plants_ferc1.revert_filled_in_float_nulls) .pipe(pudl.analysis.classify_plants_ferc1.revert_filled_in_string_nulls) .merge( - denorm_plants_utilities_ferc1, on=["utility_id_ferc1", "plant_name_ferc1"] + _out_ferc1__yearly_plants_utilities, + on=["utility_id_ferc1", "plant_name_ferc1"], ) .pipe( pudl.helpers.organize_cols, @@ -1239,17 +1296,17 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: { "root_table": "income_statement_ferc1", "table_names": [ - "income_statement_ferc1", - "depreciation_amortization_summary_ferc1", - "electric_operating_expenses_ferc1", - "electric_operating_revenues_ferc1", + "core_ferc1__yearly_income_statements_sched114", + "core_ferc1__yearly_depreciation_summary_sched336", + "core_ferc1__yearly_operating_expenses_sched320", + "core_ferc1__yearly_operating_revenues_sched300", ], "group_metric_checks": EXPLOSION_CALCULATION_TOLERANCES[ - "income_statement_ferc1" + "core_ferc1__yearly_income_statements_sched114" ], "seed_nodes": [ NodeId( - table_name="income_statement_ferc1", + table_name="core_ferc1__yearly_income_statements_sched114", xbrl_factoid="net_income_loss", utility_type="total", plant_status=pd.NA, @@ -1261,17 +1318,17 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: { "root_table": "balance_sheet_assets_ferc1", "table_names": [ - "balance_sheet_assets_ferc1", - "utility_plant_summary_ferc1", - "plant_in_service_ferc1", - "electric_plant_depreciation_functional_ferc1", + "core_ferc1__yearly_balance_sheet_assets_sched110", + "core_ferc1__yearly_utility_plant_summary_sched200", + "core_ferc1__yearly_plant_in_service_sched204", + "core_ferc1__yearly_depreciation_by_function_sched219", ], "group_metric_checks": EXPLOSION_CALCULATION_TOLERANCES[ - "balance_sheet_assets_ferc1" + "core_ferc1__yearly_balance_sheet_assets_sched110" ], "seed_nodes": [ NodeId( - table_name="balance_sheet_assets_ferc1", + table_name="core_ferc1__yearly_balance_sheet_assets_sched110", xbrl_factoid="assets_and_other_debits", utility_type="total", plant_status=pd.NA, @@ -1280,36 +1337,36 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: ], "off_by_facts": [ OffByFactoid( - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "utility_plant_in_service_classified_and_property_under_capital_leases", "electric", pd.NA, pd.NA, - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "utility_plant_in_service_completed_construction_not_classified", "electric", pd.NA, pd.NA, ), OffByFactoid( - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "utility_plant_in_service_classified_and_property_under_capital_leases", "electric", pd.NA, pd.NA, - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "utility_plant_in_service_property_under_capital_leases", "electric", pd.NA, pd.NA, ), OffByFactoid( - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "depreciation_utility_plant_in_service", "electric", pd.NA, pd.NA, - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "amortization_of_other_utility_plant_utility_plant_in_service", "electric", pd.NA, @@ -1320,15 +1377,15 @@ def create_exploded_table_assets() -> list[AssetsDefinition]: { "root_table": "balance_sheet_liabilities_ferc1", "table_names": [ - "balance_sheet_liabilities_ferc1", - "retained_earnings_ferc1", + "core_ferc1__yearly_balance_sheet_liabilities_sched110", + "core_ferc1__yearly_retained_earnings_sched118", ], "group_metric_checks": EXPLOSION_CALCULATION_TOLERANCES[ - "balance_sheet_liabilities_ferc1" + "core_ferc1__yearly_balance_sheet_liabilities_sched110" ], "seed_nodes": [ NodeId( - table_name="balance_sheet_liabilities_ferc1", + table_name="core_ferc1__yearly_balance_sheet_liabilities_sched110", xbrl_factoid="liabilities_and_other_credits", utility_type="total", plant_status=pd.NA, @@ -1921,7 +1978,8 @@ class XbrlCalculationForestFerc1(BaseModel): """A class for manipulating groups of hierarchically nested XBRL calculations. We expect that the facts reported in high-level FERC tables like - :ref:`income_statement_ferc1` and :ref:`balance_sheet_assets_ferc1` should be + :ref:`core_ferc1__yearly_income_statements_sched114` and + :ref:`core_ferc1__yearly_balance_sheet_assets_sched110` should be calculable from many individually reported granular values, based on the calculations encoded in the XBRL Metadata, and that these relationships should have a hierarchical tree structure. Several individual values from the higher level @@ -2346,21 +2404,21 @@ def forest(self: Self) -> nx.DiGraph: # only stepchildren node removal from above. a generalization here would be good almost_pure_stepparents = [ NodeId( - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "depreciation_amortization_and_depletion_utility_plant_leased_to_others", "total", pd.NA, pd.NA, ), NodeId( - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "depreciation_and_amortization_utility_plant_held_for_future_use", "total", pd.NA, pd.NA, ), NodeId( - "utility_plant_summary_ferc1", + "core_ferc1__yearly_utility_plant_summary_sched200", "utility_plant_in_service_classified_and_unclassified", "total", pd.NA, diff --git a/src/pudl/output/ferc714.py b/src/pudl/output/ferc714.py index 2a6a6bd525..426e38df5e 100644 --- a/src/pudl/output/ferc714.py +++ b/src/pudl/output/ferc714.py @@ -41,14 +41,14 @@ * `id` (int): EIA balancing authority identifier (`balancing_authority_id_eia`). * `from` (int): Reference year, to use as a template for target years. * `to` (List[int]): Target years, in the closed interval format [minimum, maximum]. - Rows in `balancing_authority_eia861` are added (if missing) for every target year + Rows in `core_eia861__yearly_balancing_authority` are added (if missing) for every target year with the attributes from the reference year. - Rows in `balancing_authority_assn_eia861` are added (or replaced, if existing) + Rows in `core_eia861__assn_balancing_authority` are added (or replaced, if existing) for every target year with the utility associations from the reference year. - Rows in `service_territory_eia861` are added (if missing) for every target year + Rows in `core_eia861__yearly_service_territory` are added (if missing) for every target year with the nearest year's associated utilities' counties. * `exclude` (Optional[List[str]]): Utilities to exclude, by state (two-letter code). - Rows are excluded from `balancing_authority_assn_eia861` with target year and state. + Rows are excluded from `core_eia861__assn_balancing_authority` with target year and state. """ UTILITIES: list[dict[str, Any]] = [ @@ -60,7 +60,7 @@ {"id": 4922, "reassign": True}, # (no code): Consumers Energy Company # NOTE: 2003-2006 parent to 40211, which is never child to parent BA (12427), - # (and 40211 never reports in service_territory_eia861) so don't reassign. + # (and 40211 never reports in core_eia861__yearly_service_territory) so don't reassign. {"id": 4254}, ] """Balancing authorities to treat as utilities in associations from EIA 861. @@ -68,13 +68,13 @@ The changes are applied locally to EIA 861 tables. * `id` (int): EIA balancing authority (BA) identifier (`balancing_authority_id_eia`). - Rows for `id` are removed from `balancing_authority_eia861`. + Rows for `id` are removed from `core_eia861__yearly_balancing_authority`. * `reassign` (Optional[bool]): Whether to reassign utilities to parent BAs. - Rows for `id` as BA in `balancing_authority_assn_eia861` are removed. + Rows for `id` as BA in `core_eia861__assn_balancing_authority` are removed. Utilities assigned to `id` for a given year are reassigned to the BAs for which `id` is an associated utility. * `replace` (Optional[bool]): Whether to remove rows where `id` is a utility in - `balancing_authority_assn_eia861`. Applies only if `reassign=True`. + `core_eia861__assn_balancing_authority`. Applies only if `reassign=True`. """ ################################################################################ @@ -171,17 +171,17 @@ def categorize_eia_code( ################################################################################ -def filled_balancing_authority_eia861( - balancing_authority_eia861: pd.DataFrame, +def filled_core_eia861__yearly_balancing_authority( + core_eia861__yearly_balancing_authority: pd.DataFrame, ) -> pd.DataFrame: - """Modified balancing_authority_eia861 table. + """Modified core_eia861__yearly_balancing_authority table. This function adds rows for each balancing authority-year pair missing from the - cleaned balancing_authority_eia861 table, using a dictionary of manual fixes. It + cleaned core_eia861__yearly_balancing_authority table, using a dictionary of manual fixes. It uses the reference year as a template. The function also removes balancing authorities that are manually categorized as utilities. """ - df = balancing_authority_eia861 + df = core_eia861__yearly_balancing_authority index = ["balancing_authority_id_eia", "report_date"] dfi = df.set_index(index) # Prepare reference rows @@ -204,18 +204,18 @@ def filled_balancing_authority_eia861( return apply_pudl_dtypes(df[~mask], group="eia") -def filled_balancing_authority_assn_eia861( - balancing_authority_assn_eia861: pd.DataFrame, +def filled_core_eia861__assn_balancing_authority( + core_eia861__assn_balancing_authority: pd.DataFrame, ) -> pd.DataFrame: - """Modified balancing_authority_assn_eia861 table. + """Modified core_eia861__assn_balancing_authority table. This function adds rows for each balancing authority-year pair missing from the - cleaned balancing_authority_assn_eia861 table, using a dictionary of manual fixes. + cleaned core_eia861__assn_balancing_authority table, using a dictionary of manual fixes. It uses the reference year as a template. The function also reassigns balancing authorities that are manually categorized as utilities to their parent balancing authorities. """ - df = balancing_authority_assn_eia861 + df = core_eia861__assn_balancing_authority # Prepare reference rows refs = [] for fix in ASSOCIATIONS: @@ -285,20 +285,22 @@ def filled_balancing_authority_assn_eia861( def filled_service_territory_eia861( - balancing_authority_assn_eia861: pd.DataFrame, - service_territory_eia861: pd.DataFrame, + core_eia861__assn_balancing_authority: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, ) -> pd.DataFrame: - """Modified service_territory_eia861 table. + """Modified core_eia861__yearly_service_territory table. This function adds rows for each balancing authority-year pair missing from the - cleaned service_territory_eia861 table, using a dictionary of manual fixes. It also + cleaned core_eia861__yearly_service_territory table, using a dictionary of manual fixes. It also drops utility-state combinations which are missing counties across all years of data, fills records missing counties with the nearest year of county data for the same utility and state. """ index = ["utility_id_eia", "state", "report_date"] # Select relevant balancing authority-utility associations - assn = filled_balancing_authority_assn_eia861(balancing_authority_assn_eia861) + assn = filled_core_eia861__assn_balancing_authority( + core_eia861__assn_balancing_authority + ) selected = np.zeros(assn.shape[0], dtype=bool) for fix in ASSOCIATIONS: years = [fix["from"], *range(fix["to"][0], fix["to"][1] + 1)] @@ -309,7 +311,7 @@ def filled_service_territory_eia861( # Reformat as unique utility-state-year assn = assn[selected][index].drop_duplicates() # Select relevant service territories - mdf = assn.merge(service_territory_eia861, how="left") + mdf = assn.merge(core_eia861__yearly_service_territory, how="left") # Drop utility-state with no counties for all years grouped = mdf.groupby(["utility_id_eia", "state"])["county_id_fips"] mdf = mdf[grouped.transform("count").gt(0)] @@ -329,7 +331,7 @@ def filled_service_territory_eia861( idx = (years - row["report_date"]).abs().idxmin() mask &= mdf["report_date"].eq(years[idx]) tables.append(mdf[mask].assign(report_date=row["report_date"])) - return pd.concat([service_territory_eia861] + tables).pipe( + return pd.concat([core_eia861__yearly_service_territory] + tables).pipe( apply_pudl_dtypes, group="eia" ) @@ -338,8 +340,9 @@ def filled_service_territory_eia861( compute_kind="Python", required_resource_keys={"dataset_settings"}, ) -def annualized_respondents_ferc714( - context, respondent_id_ferc714: pd.DataFrame +def _out_ferc714__annualized_respondents( + context, + core_ferc714__respondent_id: pd.DataFrame, ) -> pd.DataFrame: """Broadcast respondent data across all years with reported demand. @@ -351,14 +354,14 @@ def annualized_respondents_ferc714( having entries for years in which they reported no demand, and that's fine. They can be filtered later. """ - if "report_date" in respondent_id_ferc714.columns: + if "report_date" in core_ferc714__respondent_id.columns: raise AssertionError("report_date already present, can't be added again!") ferc714_settings = context.resources.dataset_settings.ferc714 report_dates = pd.DataFrame( {"report_date": pd.to_datetime(sorted(ferc714_settings.years), format="%Y")} ) - return respondent_id_ferc714.merge(report_dates, how="cross") + return core_ferc714__respondent_id.merge(report_dates, how="cross") @asset( @@ -375,13 +378,13 @@ def annualized_respondents_ferc714( }, compute_kind="Python", ) -def categorized_respondents_ferc714( +def _out_ferc714__categorized_respondents( context, - respondent_id_ferc714: pd.DataFrame, - denorm_utilities_eia: pd.DataFrame, - service_territory_eia861: pd.DataFrame, - balancing_authority_eia861: pd.DataFrame, - annualized_respondents_ferc714: pd.DataFrame, + core_ferc714__respondent_id: pd.DataFrame, + out_eia__yearly_utilities: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, + core_eia861__yearly_balancing_authority: pd.DataFrame, + _out_ferc714__annualized_respondents: pd.DataFrame, ) -> pd.DataFrame: """Annualized respondents with ``respondent_type`` assigned if possible. @@ -395,13 +398,15 @@ def categorized_respondents_ferc714( logger.info("Categorizing EIA codes associated with FERC-714 Respondents.") - bal_auth = filled_balancing_authority_eia861(balancing_authority_eia861) + bal_auth = filled_core_eia861__yearly_balancing_authority( + core_eia861__yearly_balancing_authority + ) utilids_all_eia = utility_ids_all_eia( - denorm_utilities_eia, service_territory_eia861 + out_eia__yearly_utilities, core_eia861__yearly_service_territory ) categorized = categorize_eia_code( - respondent_id_ferc714.eia_code.dropna().unique(), + core_ferc714__respondent_id.eia_code.dropna().unique(), ba_ids=bal_auth.balancing_authority_id_eia.dropna().unique(), util_ids=utilids_all_eia.utility_id_eia, priority=priority, @@ -409,7 +414,9 @@ def categorized_respondents_ferc714( logger.info( "Merging categorized EIA codes with annualized FERC-714 Respondent data." ) - categorized = pd.merge(categorized, annualized_respondents_ferc714, how="right") + categorized = pd.merge( + categorized, _out_ferc714__annualized_respondents, how="right" + ) # Names, ids, and codes for BAs identified as FERC 714 respondents # NOTE: this is not *strictly* correct, because the EIA BAs are not # eternal and unchanging. There's at least one case in which the BA @@ -479,12 +486,12 @@ def categorized_respondents_ferc714( compute_kind="Python", io_manager_key="pudl_sqlite_io_manager", ) -def fipsified_respondents_ferc714( +def out_ferc714__respondents_with_fips( context, - categorized_respondents_ferc714: pd.DataFrame, - balancing_authority_assn_eia861: pd.DataFrame, - service_territory_eia861: pd.DataFrame, - utility_assn_eia861: pd.DataFrame, + _out_ferc714__categorized_respondents: pd.DataFrame, + core_eia861__assn_balancing_authority: pd.DataFrame, + core_eia861__yearly_service_territory: pd.DataFrame, + core_eia861__assn_utility: pd.DataFrame, ) -> pd.DataFrame: """Annual respondents with the county FIPS IDs for their service territories. @@ -495,24 +502,28 @@ def fipsified_respondents_ferc714( are thousands of counties and many of them are served by more than one entity. Currently respondents categorized as ``utility`` will include any county that - appears in the ``service_territory_eia861`` table in association with that + appears in the ``core_eia861__yearly_service_territory`` table in association with that utility ID in each year, while for ``balancing_authority`` respondents, some counties can be excluded based on state (if ``limit_by_state==True``). """ # - assn = filled_balancing_authority_assn_eia861(balancing_authority_assn_eia861) + assn = filled_core_eia861__assn_balancing_authority( + core_eia861__assn_balancing_authority + ) st_eia861 = filled_service_territory_eia861( - balancing_authority_assn_eia861, service_territory_eia861 + core_eia861__assn_balancing_authority, core_eia861__yearly_service_territory ) # Generate the BA:FIPS relation: ba_counties = pd.merge( - categorized_respondents_ferc714.query("respondent_type=='balancing_authority'"), + _out_ferc714__categorized_respondents.query( + "respondent_type=='balancing_authority'" + ), pudl.analysis.service_territory.get_territory_fips( - ids=categorized_respondents_ferc714.balancing_authority_id_eia.unique(), + ids=_out_ferc714__categorized_respondents.balancing_authority_id_eia.unique(), assn=assn, assn_col="balancing_authority_id_eia", - service_territory_eia861=st_eia861, + core_eia861__yearly_service_territory=st_eia861, limit_by_state=context.op_config["limit_by_state"], ), on=["report_date", "balancing_authority_id_eia"], @@ -520,12 +531,12 @@ def fipsified_respondents_ferc714( ) # Generate the Util:FIPS relation: util_counties = pd.merge( - categorized_respondents_ferc714.query("respondent_type=='utility'"), + _out_ferc714__categorized_respondents.query("respondent_type=='utility'"), pudl.analysis.service_territory.get_territory_fips( - ids=categorized_respondents_ferc714.utility_id_eia.unique(), - assn=utility_assn_eia861, + ids=_out_ferc714__categorized_respondents.utility_id_eia.unique(), + assn=core_eia861__assn_utility, assn_col="utility_id_eia", - service_territory_eia861=st_eia861, + core_eia861__yearly_service_territory=st_eia861, limit_by_state=context.op_config["limit_by_state"], ), on=["report_date", "utility_id_eia"], @@ -535,8 +546,8 @@ def fipsified_respondents_ferc714( [ ba_counties, util_counties, - categorized_respondents_ferc714[ - categorized_respondents_ferc714.respondent_type.isnull() + _out_ferc714__categorized_respondents[ + _out_ferc714__categorized_respondents.respondent_type.isnull() ], ] ).pipe(apply_pudl_dtypes) @@ -544,8 +555,9 @@ def fipsified_respondents_ferc714( @asset(compute_kind="Python") -def georeferenced_counties_ferc714( - fipsified_respondents_ferc714: pd.DataFrame, county_censusdp1: gpd.GeoDataFrame +def _out_ferc714__georeferenced_counties( + out_ferc714__respondents_with_fips: pd.DataFrame, + core_censusdp1__entity_county: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """Annual respondents with all associated county-level geometries. @@ -556,16 +568,16 @@ def georeferenced_counties_ferc714( of the FIPS IDs so you can also still do ID based analyses. """ counties_gdf = pudl.analysis.service_territory.add_geometries( - fipsified_respondents_ferc714, census_gdf=county_censusdp1 + out_ferc714__respondents_with_fips, census_gdf=core_censusdp1__entity_county ).pipe(apply_pudl_dtypes) return counties_gdf @asset(compute_kind="Python") -def georeferenced_respondents_ferc714( - fipsified_respondents_ferc714: pd.DataFrame, - summarized_demand_ferc714: pd.DataFrame, - county_censusdp1: gpd.GeoDataFrame, +def _out_ferc714__georeferenced_respondents( + out_ferc714__respondents_with_fips: pd.DataFrame, + out_ferc714__summarized_demand: pd.DataFrame, + core_censusdp1__entity_county: gpd.GeoDataFrame, ) -> gpd.GeoDataFrame: """Annual respondents with a single all-encompassing geometry for each year. @@ -579,13 +591,13 @@ def georeferenced_respondents_ferc714( """ respondents_gdf = ( pudl.analysis.service_territory.add_geometries( - fipsified_respondents_ferc714, - census_gdf=county_censusdp1, + out_ferc714__respondents_with_fips, + census_gdf=core_censusdp1__entity_county, dissolve=True, dissolve_by=["report_date", "respondent_id_ferc714"], ) .merge( - summarized_demand_ferc714[ + out_ferc714__summarized_demand[ ["report_date", "respondent_id_ferc714", "demand_annual_mwh"] ] ) @@ -595,11 +607,12 @@ def georeferenced_respondents_ferc714( @asset(compute_kind="Python", io_manager_key="pudl_sqlite_io_manager") -def summarized_demand_ferc714( - annualized_respondents_ferc714: pd.DataFrame, - demand_hourly_pa_ferc714: pd.DataFrame, - categorized_respondents_ferc714: pd.DataFrame, - georeferenced_counties_ferc714: gpd.GeoDataFrame, +def out_ferc714__summarized_demand( + _out_ferc714__annualized_respondents: pd.DataFrame, + core_ferc714__hourly_demand_pa: pd.DataFrame, + out_ferc714__respondents_with_fips: pd.DataFrame, + _out_ferc714__categorized_respondents: pd.DataFrame, + _out_ferc714__georeferenced_counties: gpd.GeoDataFrame, ) -> pd.DataFrame: """Compile annualized, categorized respondents and summarize values. @@ -614,8 +627,8 @@ def summarized_demand_ferc714( """ demand_annual = ( pd.merge( - annualized_respondents_ferc714, - demand_hourly_pa_ferc714.loc[ + _out_ferc714__annualized_respondents, + core_ferc714__hourly_demand_pa.loc[ :, ["report_date", "respondent_id_ferc714", "demand_mwh"] ], on=["report_date", "respondent_id_ferc714"], @@ -627,7 +640,7 @@ def summarized_demand_ferc714( .sum(min_count=1) .rename(columns={"demand_mwh": "demand_annual_mwh"}) .merge( - georeferenced_counties_ferc714.groupby( + _out_ferc714__georeferenced_counties.groupby( ["report_date", "respondent_id_ferc714"], as_index=False )[["population", "area_km2"]].sum(min_count=1), on=["report_date", "respondent_id_ferc714"], @@ -641,6 +654,6 @@ def summarized_demand_ferc714( ) # Merge respondent categorizations into the annual demand demand_summary = pd.merge( - demand_annual, categorized_respondents_ferc714, how="left" + demand_annual, _out_ferc714__categorized_respondents, how="left" ).pipe(apply_pudl_dtypes) return demand_summary diff --git a/src/pudl/output/pudltabl.py b/src/pudl/output/pudltabl.py index fe04e62a2a..48df48b037 100644 --- a/src/pudl/output/pudltabl.py +++ b/src/pudl/output/pudltabl.py @@ -77,9 +77,9 @@ def __init__( data with state-fuel averages from EIA's bulk electricity data. roll_fuel_cost: if True, apply a rolling average to a subset of output table's columns (currently only ``fuel_cost_per_mmbtu`` - for the ``fuel_receipts_costs_eia923`` table.) + for the ``core_eia923__monthly_fuel_receipts_costs`` table.) fill_net_gen: if True, use the net generation from the - generation_fuel_eia923 - which is reported at the + core_eia923__monthly_generation_fuel - which is reported at the plant/fuel/prime mover level and re-allocated to generators in ``mcoe()``, ``capacity_factor()`` and ``heat_rate_by_unit()``. fill_tech_desc: If True, fill the technology_description @@ -89,6 +89,11 @@ def __init__( unit_ids: If True, use several heuristics to assign individual generators to functional units. EXPERIMENTAL. """ + logger.warning( + "PudlTabl is deprecated and will be removed from the pudl package " + "once known users have migrated to accessing the data directly from " + "pudl.sqlite. " + ) if not isinstance(pudl_engine, sa.engine.base.Engine): raise TypeError( "PudlTabl needs pudl_engine to be a SQLAlchemy Engine, but we " @@ -133,110 +138,110 @@ def _register_output_methods(self: Self): # PudlTabl will generate a method to read each table from the DB with the given method name table_method_map_any_freq = { # table_name: method_name # denorm_ferc1 - "denorm_balance_sheet_assets_ferc1": "denorm_balance_sheet_assets_ferc1", - "denorm_balance_sheet_liabilities_ferc1": "denorm_balance_sheet_liabilities_ferc1", - "denorm_cash_flow_ferc1": "denorm_cash_flow_ferc1", - "denorm_depreciation_amortization_summary_ferc1": "denorm_depreciation_amortization_summary_ferc1", - "denorm_electric_energy_dispositions_ferc1": "denorm_electric_energy_dispositions_ferc1", - "denorm_electric_energy_sources_ferc1": "denorm_electric_energy_sources_ferc1", - "denorm_electric_operating_expenses_ferc1": "denorm_electric_operating_expenses_ferc1", - "denorm_electric_operating_revenues_ferc1": "denorm_electric_operating_revenues_ferc1", - "denorm_electric_plant_depreciation_changes_ferc1": "denorm_electric_plant_depreciation_changes_ferc1", - "denorm_electric_plant_depreciation_functional_ferc1": "denorm_electric_plant_depreciation_functional_ferc1", - "denorm_electricity_sales_by_rate_schedule_ferc1": "denorm_electricity_sales_by_rate_schedule_ferc1", - "denorm_income_statement_ferc1": "denorm_income_statement_ferc1", - "denorm_other_regulatory_liabilities_ferc1": "denorm_other_regulatory_liabilities_ferc1", - "denorm_retained_earnings_ferc1": "denorm_retained_earnings_ferc1", - "denorm_transmission_statistics_ferc1": "denorm_transmission_statistics_ferc1", - "denorm_utility_plant_summary_ferc1": "denorm_utility_plant_summary_ferc1", - "denorm_plants_utilities_ferc1": "pu_ferc1", - "denorm_plants_steam_ferc1": "plants_steam_ferc1", - "denorm_fuel_ferc1": "fuel_ferc1", - "denorm_fuel_by_plant_ferc1": "fbp_ferc1", - "denorm_plants_small_ferc1": "plants_small_ferc1", - "denorm_plants_hydro_ferc1": "plants_hydro_ferc1", - "denorm_plants_pumped_storage_ferc1": "plants_pumped_storage_ferc1", - "denorm_purchased_power_ferc1": "purchased_power_ferc1", - "denorm_plant_in_service_ferc1": "plant_in_service_ferc1", - "denorm_plants_all_ferc1": "plants_all_ferc1", + "out_ferc1__yearly_balance_sheet_assets_sched110": "denorm_balance_sheet_assets_ferc1", + "out_ferc1__yearly_balance_sheet_liabilities_sched110": "denorm_balance_sheet_liabilities_ferc1", + "out_ferc1__yearly_cash_flows_sched120": "denorm_cash_flow_ferc1", + "out_ferc1__yearly_depreciation_summary_sched336": "denorm_depreciation_amortization_summary_ferc1", + "out_ferc1__yearly_energy_dispositions_sched401": "denorm_electric_energy_dispositions_ferc1", + "out_ferc1__yearly_energy_sources_sched401": "denorm_electric_energy_sources_ferc1", + "out_ferc1__yearly_operating_expenses_sched320": "denorm_electric_operating_expenses_ferc1", + "out_ferc1__yearly_operating_revenues_sched300": "denorm_electric_operating_revenues_ferc1", + "out_ferc1__yearly_depreciation_changes_sched219": "denorm_electric_plant_depreciation_changes_ferc1", + "out_ferc1__yearly_depreciation_by_function_sched219": "denorm_electric_plant_depreciation_functional_ferc1", + "out_ferc1__yearly_sales_by_rate_schedules_sched304": "denorm_electricity_sales_by_rate_schedule_ferc1", + "out_ferc1__yearly_income_statements_sched114": "denorm_income_statement_ferc1", + "out_ferc1__yearly_other_regulatory_liabilities_sched278": "denorm_other_regulatory_liabilities_ferc1", + "out_ferc1__yearly_retained_earnings_sched118": "denorm_retained_earnings_ferc1", + "out_ferc1__yearly_transmission_lines_sched422": "denorm_transmission_statistics_ferc1", + "out_ferc1__yearly_utility_plant_summary_sched200": "denorm_utility_plant_summary_ferc1", + "_out_ferc1__yearly_plants_utilities": "pu_ferc1", + "_out_ferc1__yearly_steam_plants_sched402": "plants_steam_ferc1", + "out_ferc1__yearly_steam_plants_fuel_sched402": "fuel_ferc1", + "out_ferc1__yearly_steam_plants_fuel_by_plant_sched402": "fbp_ferc1", + "_out_ferc1__yearly_small_plants_sched410": "plants_small_ferc1", + "_out_ferc1__yearly_hydroelectric_plants_sched406": "plants_hydro_ferc1", + "_out_ferc1__yearly_pumped_storage_plants_sched408": "plants_pumped_storage_ferc1", + "out_ferc1__yearly_purchased_power_and_exchanges_sched326": "purchased_power_ferc1", + "out_ferc1__yearly_plant_in_service_sched204": "plant_in_service_ferc1", + "out_ferc1__yearly_all_plants": "plants_all_ferc1", # denorm_eia (data comes from multiple EIA forms) - "denorm_plants_eia": "plants_eia860", - "denorm_utilities_eia": "utils_eia860", - "denorm_boilers_eia": "boil_eia860", - "denorm_generators_eia": "gens_eia860", - "denorm_plants_utilities_eia": "pu_eia860", + "out_eia__yearly_plants": "plants_eia860", + "out_eia__yearly_utilities": "utils_eia860", + "out_eia__yearly_boilers": "boil_eia860", + "_out_eia__yearly_generators": "gens_eia860", + "_out_eia__plants_utilities": "pu_eia860", # eia860 (denormalized, data primarily from EIA-860) - "denorm_ownership_eia860": "own_eia860", - "boiler_generator_assn_eia860": "bga_eia860", - "denorm_emissions_control_equipment_eia860": "denorm_emissions_control_equipment_eia860", - "boiler_emissions_control_equipment_assn_eia860": "boiler_emissions_control_equipment_assn_eia860", - "emissions_control_equipment_eia860": "emissions_control_equipment_eia860", - "boiler_stack_flue_assn_eia860": "boiler_stack_flue_assn_eia860", - "boiler_cooling_assn_eia860": "boiler_cooling_assn_eia860", + "out_eia860__yearly_ownership": "own_eia860", + "core_eia860__assn_boiler_generator": "bga_eia860", + "out_eia860__yearly_emissions_control_equipment": "denorm_emissions_control_equipment_eia860", + "core_eia860__assn_yearly_boiler_emissions_control_equipment": "boiler_emissions_control_equipment_assn_eia860", + "core_eia860__scd_emissions_control_equipment": "emissions_control_equipment_eia860", + "core_eia860__assn_boiler_stack_flue": "boiler_stack_flue_assn_eia860", + "core_eia860__assn_boiler_cooling": "boiler_cooling_assn_eia860", # eia861 (clean) - "service_territory_eia861": "service_territory_eia861", - "sales_eia861": "sales_eia861", - "advanced_metering_infrastructure_eia861": "advanced_metering_infrastructure_eia861", - "demand_response_eia861": "demand_response_eia861", - "demand_response_water_heater_eia861": "demand_response_water_heater_eia861", - "demand_side_management_sales_eia861": "demand_side_management_sales_eia861", - "demand_side_management_ee_dr_eia861": "demand_side_management_ee_dr_eia861", - "demand_side_management_misc_eia861": "demand_side_management_misc_eia861", - "distributed_generation_tech_eia861": "distributed_generation_tech_eia861", - "distributed_generation_fuel_eia861": "distributed_generation_fuel_eia861", - "distributed_generation_misc_eia861": "distributed_generation_misc_eia861", - "distribution_systems_eia861": "distribution_systems_eia861", - "dynamic_pricing_eia861": "dynamic_pricing_eia861", - "energy_efficiency_eia861": "energy_efficiency_eia861", - "green_pricing_eia861": "green_pricing_eia861", - "mergers_eia861": "mergers_eia861", - "net_metering_customer_fuel_class_eia861": "net_metering_customer_fuel_class_eia861", - "net_metering_misc_eia861": "net_metering_misc_eia861", - "non_net_metering_customer_fuel_class_eia861": "non_net_metering_customer_fuel_class_eia861", - "non_net_metering_misc_eia861": "non_net_metering_misc_eia861", - "operational_data_revenue_eia861": "operational_data_revenue_eia861", - "operational_data_misc_eia861": "operational_data_misc_eia861", - "reliability_eia861": "reliability_eia861", - "utility_data_nerc_eia861": "utility_data_nerc_eia861", - "utility_data_rto_eia861": "utility_data_rto_eia861", - "utility_data_misc_eia861": "utility_data_misc_eia861", - "utility_assn_eia861": "utility_assn_eia861", - "balancing_authority_eia861": "balancing_authority_eia861", - "balancing_authority_assn_eia861": "balancing_authority_assn_eia861", + "core_eia861__yearly_service_territory": "service_territory_eia861", + "core_eia861__yearly_sales": "sales_eia861", + "core_eia861__yearly_advanced_metering_infrastructure": "advanced_metering_infrastructure_eia861", + "core_eia861__yearly_demand_response": "demand_response_eia861", + "core_eia861__yearly_demand_response_water_heater": "demand_response_water_heater_eia861", + "core_eia861__yearly_demand_side_management_sales": "demand_side_management_sales_eia861", + "core_eia861__yearly_demand_side_management_ee_dr": "demand_side_management_ee_dr_eia861", + "core_eia861__yearly_demand_side_management_misc": "demand_side_management_misc_eia861", + "core_eia861__yearly_distributed_generation_tech": "distributed_generation_tech_eia861", + "core_eia861__yearly_distributed_generation_fuel": "distributed_generation_fuel_eia861", + "core_eia861__yearly_distributed_generation_misc": "distributed_generation_misc_eia861", + "core_eia861__yearly_distribution_systems": "distribution_systems_eia861", + "core_eia861__yearly_dynamic_pricing": "dynamic_pricing_eia861", + "core_eia861__yearly_energy_efficiency": "energy_efficiency_eia861", + "core_eia861__yearly_green_pricing": "green_pricing_eia861", + "core_eia861__yearly_mergers": "mergers_eia861", + "core_eia861__yearly_net_metering_customer_fuel_class": "net_metering_customer_fuel_class_eia861", + "core_eia861__yearly_net_metering_misc": "net_metering_misc_eia861", + "core_eia861__yearly_non_net_metering_customer_fuel_class": "non_net_metering_customer_fuel_class_eia861", + "core_eia861__yearly_non_net_metering_misc": "non_net_metering_misc_eia861", + "core_eia861__yearly_operational_data_revenue": "operational_data_revenue_eia861", + "core_eia861__yearly_operational_data_misc": "operational_data_misc_eia861", + "core_eia861__yearly_reliability": "reliability_eia861", + "core_eia861__yearly_utility_data_nerc": "utility_data_nerc_eia861", + "core_eia861__yearly_utility_data_rto": "utility_data_rto_eia861", + "core_eia861__yearly_utility_data_misc": "utility_data_misc_eia861", + "core_eia861__assn_utility": "core_eia861__assn_utility", + "core_eia861__yearly_balancing_authority": "balancing_authority_eia861", + "core_eia861__assn_balancing_authority": "balancing_authority_assn_eia861", # eia923 (denormalized, data primarily from EIA-923) - "denorm_boiler_fuel_AGG_eia923": "bf_eia923", - "denorm_fuel_receipts_costs_AGG_eia923": "frc_eia923", - "denorm_generation_AGG_eia923": "gen_original_eia923", - "denorm_generation_fuel_combined_AGG_eia923": "gf_eia923", + "out_eia923__AGG_boiler_fuel": "bf_eia923", + "out_eia923__AGG_fuel_receipts_costs": "frc_eia923", + "out_eia923__AGG_generation": "gen_original_eia923", + "out_eia923__AGG_generation_fuel_combined": "gf_eia923", # ferc714 - "respondent_id_ferc714": "respondent_id_ferc714", - "demand_hourly_pa_ferc714": "demand_hourly_pa_ferc714", - "fipsified_respondents_ferc714": "fipsified_respondents_ferc714", - "summarized_demand_ferc714": "summarized_demand_ferc714", + "core_ferc714__respondent_id": "respondent_id_ferc714", + "core_ferc714__hourly_demand_pa": "demand_hourly_pa_ferc714", + "out_ferc714__respondents_with_fips": "fipsified_respondents_ferc714", + "out_ferc714__summarized_demand": "summarized_demand_ferc714", # service territory - "compiled_geometry_balancing_authority_eia861": "compiled_geometry_balancing_authority_eia861", - "compiled_geometry_utility_eia861": "compiled_geometry_utility_eia861", + "out_eia861__compiled_geometry_balancing_authorities": "compiled_geometry_balancing_authority_eia861", + "out_eia861__compiled_geometry_utilities": "compiled_geometry_utility_eia861", # state demand - "predicted_state_hourly_demand": "predicted_state_hourly_demand", + "out_ferc714__hourly_predicted_state_demand": "predicted_state_hourly_demand", # plant parts - "mega_generators_eia": "gens_mega_eia", - "plant_parts_eia": "plant_parts_eia", - "out__yearly_plants_all_ferc1_plant_parts_eia": "ferc1_eia", + "out_eia__yearly_generators_by_ownership": "gens_mega_eia", + "out_eia__yearly_plant_parts": "plant_parts_eia", + "out_pudl__yearly_assn_eia_ferc1_plant_parts": "ferc1_eia", } table_method_map_any_agg = { - "generation_fuel_by_generator_energy_source_AGG_eia923": "gen_fuel_by_generator_energy_source_eia923", - "generation_fuel_by_generator_AGG_eia923": "gen_fuel_by_generator_eia923", - "heat_rate_by_unit_AGG": "hr_by_unit", - "heat_rate_by_generator_AGG": "hr_by_gen", - "capacity_factor_by_generator_AGG": "capacity_factor", - "fuel_cost_by_generator_AGG": "fuel_cost", - "mcoe_AGG": "mcoe", - "mcoe_generators_AGG": "mcoe_generators", + "out_eia923__AGG_generation_fuel_by_generator_energy_source": "gen_fuel_by_generator_energy_source_eia923", + "out_eia923__AGG_generation_fuel_by_generator": "gen_fuel_by_generator_eia923", + "_out_eia__AGG_heat_rate_by_unit": "hr_by_unit", + "_out_eia__AGG_heat_rate_by_generator": "hr_by_gen", + "_out_eia__AGG_capacity_factor_by_generator": "capacity_factor", + "_out_eia__AGG_fuel_cost_by_generator": "fuel_cost", + "_out_eia__AGG_derived_generator_attributes": "mcoe", + "out_eia__AGG_generators": "mcoe_generators", } table_method_map_yearly_only = { - "generation_fuel_by_generator_energy_source_owner_yearly_eia923": "gen_fuel_by_generator_energy_source_owner_eia923", + "out_eia923__yearly_generation_fuel_by_generator_energy_source_owner": "gen_fuel_by_generator_energy_source_owner_eia923", } for table_name, method_name in ( @@ -296,6 +301,12 @@ def _get_table_from_db( "It is retained for backwards compatibility only." ) table_name = self._agg_table_name(table_name) + logger.warning( + "PudlTabl is deprecated and will be removed from the pudl package " + "once known users have migrated to accessing the data directly from " + "pudl.sqlite. To access the data returned by this method, " + f"use the {table_name} table in the pudl.sqlite database." + ) resource = Resource.from_id(table_name) return pd.concat( [ @@ -374,14 +385,14 @@ def gen_eia923(self: Self, update: bool = False) -> pd.DataFrame: """Pull EIA 923 net generation data by generator. Net generation is reported in two seperate tables in EIA 923: in the - generation_eia923 and generation_fuel_eia923 tables. While the - generation_fuel_eia923 table is more complete (the generation_eia923 - table includes only ~55% of the reported MWhs), the generation_eia923 + core_eia923__monthly_generation and core_eia923__monthly_generation_fuel tables. While the + core_eia923__monthly_generation_fuel table is more complete (the core_eia923__monthly_generation + table includes only ~55% of the reported MWhs), the core_eia923__monthly_generation table is more granular (it is reported at the generator level). - This method either grabs the generation_eia923 table that is reported + This method either grabs the core_eia923__monthly_generation table that is reported by generator, or allocates net generation from the - generation_fuel_eia923 table to the generator level. + core_eia923__monthly_generation_fuel table to the generator level. Args: update: Ignored. Retained for backwards compatibility only. @@ -395,12 +406,14 @@ def gen_eia923(self: Self, update: bool = False) -> pd.DataFrame: "Allocated net generation requires frequency of `AS` or `MS`, " f"got {self.freq}" ) - table_name = self._agg_table_name("generation_fuel_by_generator_AGG_eia923") + table_name = self._agg_table_name( + "out_eia923__AGG_generation_fuel_by_generator" + ) gen_df = self._get_table_from_db(table_name) resource = Resource.from_id(table_name) gen_df = gen_df.loc[:, resource.get_field_names()] else: - table_name = self._agg_table_name("denorm_generation_AGG_eia923") + table_name = self._agg_table_name("out_eia923__AGG_generation") gen_df = self._get_table_from_db(table_name) return gen_df @@ -409,6 +422,6 @@ def gen_eia923(self: Self, update: bool = False) -> pd.DataFrame: ########################################################################### def epacamd_eia(self: Self) -> pd.DataFrame: """Read the EPACAMD-EIA Crosswalk from the PUDL DB.""" - return pd.read_sql("epacamd_eia", self.pudl_engine).pipe( + return pd.read_sql("core_epa__assn_eia_epacamd", self.pudl_engine).pipe( apply_pudl_dtypes, group="glue" ) diff --git a/src/pudl/output/sql/denorm_plants_utilities_ferc1.sql b/src/pudl/output/sql/denorm_plants_utilities_ferc1.sql index f939e3202f..b47ca7e6c8 100644 --- a/src/pudl/output/sql/denorm_plants_utilities_ferc1.sql +++ b/src/pudl/output/sql/denorm_plants_utilities_ferc1.sql @@ -1,5 +1,5 @@ -- Build a view of useful FERC Plant & Utility information. CREATE VIEW denorm_plants_utilities_ferc1 AS SELECT * -FROM plants_ferc1 - INNER JOIN utilities_ferc1 USING(utility_id_ferc1); +FROM core_pudl__assn_ferc1_pudl_plants + INNER JOIN core_pudl__assn_ferc1_pudl_utilities USING(utility_id_ferc1); diff --git a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv index a39c371bc8..5885113f61 100644 --- a/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv +++ b/src/pudl/package_data/ferc1/xbrl_calculation_component_fixes.csv @@ -1,148 +1,148 @@ -table_name_parent,xbrl_factoid_parent,table_name,xbrl_factoid,utility_type,plant_status,plant_function,weight -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,abandonment_of_leases,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,total,,,1.0 -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_other_utility_plant_utility_plant_in_service,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,amortization_of_plant_acquisition_adjustment,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_leased_to_others,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_and_amortization_utility_plant_held_for_future_use,,,, -balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_utility_plant_in_service,,,, -balance_sheet_assets_ferc1,construction_work_in_progress,utility_plant_summary_ferc1,construction_work_in_progress,total,,,1.0 -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_hedges_long_term,,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,derivative_instrument_assets_long_term,,,, -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_hedges_long_term,total,,,-1.0 -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_derivative_instrument_assets_long_term,total,,,-1.0 -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,less_noncurrent_portion_of_allowances,total,,,-1.0 -balance_sheet_assets_ferc1,current_and_accrued_assets,balance_sheet_assets_ferc1,noncurrent_portion_of_allowances,,,, -balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_survey_and_investigation_charges,total,,,1.0 -balance_sheet_assets_ferc1,deferred_debits,balance_sheet_assets_ferc1,preliminary_natural_gas_and_other_survey_and_investigation_charges,total,,,1.0 -balance_sheet_assets_ferc1,nuclear_fuel_net,balance_sheet_assets_ferc1,nuclear_fuel,total,,,1.0 -balance_sheet_assets_ferc1,nuclear_fuel_net,nuclear_fuel_materials_ferc1,nuclear_fuel_materials_and_assemblies,,,, -balance_sheet_assets_ferc1,nuclear_fuel_net,nuclear_fuel_materials_ferc1,spent_nuclear_fuel,,,, -balance_sheet_assets_ferc1,other_property_and_investments,balance_sheet_assets_ferc1,special_funds_all,total,,,1.0 -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,construction_work_in_progress,,,, -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,utility_plant,,,, -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_acquisition_adjustment,,,, -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,total,,,1.0 -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_held_for_future_use,,,, -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,,,, -balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,utility_plant_summary_ferc1,utility_plant_leased_to_others,,,, -balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,,,, -balance_sheet_assets_ferc1,utility_plant_net,balance_sheet_assets_ferc1,utility_plant_and_construction_work_in_progress,,,, -balance_sheet_assets_ferc1,utility_plant_net,utility_plant_summary_ferc1,utility_plant_net,total,,,1.0 -balance_sheet_liabilities_ferc1,deferred_credits,balance_sheet_liabilities_ferc1,accumulated_deferred_income_taxes,total,,,1.0 -balance_sheet_liabilities_ferc1,retained_earnings,retained_earnings_ferc1,retained_earnings,total,,,1.0 -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities,,,, -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities,total,,,-1.0 -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,long_term_portion_of_derivative_instrument_liabilities_hedges,,,, -balance_sheet_liabilities_ferc1,current_and_accrued_liabilities,balance_sheet_liabilities_ferc1,less_long_term_portion_of_derivative_instrument_liabilities_hedges,total,,,-1.0 -balance_sheet_liabilities_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,total,,,1.0 -electric_energy_sources_ferc1,sources_of_energy,electric_energy_sources_ferc1,megawatt_hours_purchased,,,,1.0 -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,electric_expenses_hydraulic_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,hydraulic_expenses,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,maintenance_of_electric_plant_hydraulic_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,maintenance_of_miscellaneous_hydraulic_plant,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,maintenance_of_reservoirs_dams_and_waterways,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,maintenance_of_structures_hydraulic_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,maintenance_supervision_and_engineering_hydraulic_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,miscellaneous_hydraulic_power_generation_expenses,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,rents_hydraulic_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,electric_operating_expenses_ferc1,water_for_power,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,plants_hydro_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,plants_pumped_storage_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,power_production_expenses_hydraulic_power,plants_steam_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,allowances,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,coolants_and_water,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,electric_expenses_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,fuel_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,maintenance_of_boiler_plant_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,maintenance_of_electric_plant_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,maintenance_of_miscellaneous_steam_plant,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,maintenance_of_structures_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,maintenance_supervision_and_engineering_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,miscellaneous_steam_power_expenses,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,rents_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,steam_expenses_steam_power_generation,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,steam_from_other_sources,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,electric_operating_expenses_ferc1,steam_transferred_credit,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_hydro_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_pumped_storage_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,power_production_expenses_steam_power,plants_steam_ferc1,opex_operations,,,, -electric_operating_expenses_ferc1,transmission_operation_expense,electric_operating_expenses_ferc1,load_dispatching_transmission_expense,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,forfeited_discounts,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,interdepartmental_rents,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_revenue,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,miscellaneous_service_revenues,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_electric_revenue,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,other_miscellaneous_operating_revenues,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,regional_transmission_service_revenues,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,rent_from_electric_property,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,revenues_from_transmission_of_electricity_of_others,electric,,,1.0 -electric_operating_revenues_ferc1,other_operating_revenues,electric_operating_revenues_ferc1,sales_of_water_and_water_power,electric,,,1.0 -electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,large_or_industrial,electric,,,1.0 -electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electric_operating_revenues_ferc1,small_or_commercial,electric,,,1.0 -electric_operating_revenues_ferc1,sales_to_ultimate_consumers,electricity_sales_by_rate_schedule_ferc1,commercial_and_industrial,,,, -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,book_cost_of_asset_retirement_costs,electric,,,1.0 -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,depreciation_provision,electric,,,1.0 -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,net_charges_for_retired_plant,electric,,,1.0 -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,other_adjustments_to_accumulated_depreciation,electric,,,1.0 -electric_plant_depreciation_changes_ferc1,ending_balance,electric_plant_depreciation_changes_ferc1,starting_balance,electric,,,1.0 -income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_limited_term_electric_plant,electric,,total,1.0 -income_statement_ferc1,amortization_and_depletion_of_utility_plant,depreciation_amortization_summary_ferc1,amortization_other_electric_plant,electric,,total,1.0 -income_statement_ferc1,depreciation_expense,depreciation_amortization_summary_ferc1,depreciation_expense,electric,,total,1.0 -income_statement_ferc1,depreciation_expense_for_asset_retirement_costs,depreciation_amortization_summary_ferc1,depreciation_expense_asset_retirement,electric,,total,1.0 -income_statement_ferc1,income_before_extraordinary_items,income_statement_ferc1,net_utility_operating_income,,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,distribution_maintenance_expense_electric,electric,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_maintenance_expense,electric,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,maintenance_of_general_plant,electric,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,nuclear_power_generation_maintenance_expense,electric,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,other_power_generation_maintenance_expense,electric,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,regional_market_maintenance_expense,electric,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,steam_power_generation_maintenance_expense,electric,,,1.0 -income_statement_ferc1,maintenance_expense,electric_operating_expenses_ferc1,transmission_maintenance_expense_electric,electric,,,1.0 -income_statement_ferc1,operating_revenues,electric_operating_revenues_ferc1,electric_operating_revenues,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,administrative_and_general_operation_expense,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_account_expenses,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,customer_service_and_information_expenses,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,distribution_operation_expenses_electric,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,hydraulic_power_generation_operations_expense,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,nuclear_power_generation_operations_expense,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,other_power_generation_operations_expense,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,regional_market_operation_expense,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,sales_expenses,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,steam_power_generation_operations_expense,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,transmission_operation_expense,electric,,,1.0 -income_statement_ferc1,operation_expense,electric_operating_expenses_ferc1,power_production_expenses,electric,,,1.0 -income_statement_ferc1,other_income_deductions,income_statement_ferc1,miscellaneous_deductions,,,,1.0 -income_statement_ferc1,taxes_on_other_income_and_deductions,income_statement_ferc1,investment_tax_credits,,,,-1.0 -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,distribution_plant,electric,,,1.0 -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,general_plant,electric,,,1.0 -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,intangible_plant,electric,,,1.0 -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,production_plant,electric,,,1.0 -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,electric,,,1.0 -plant_in_service_ferc1,electric_plant_in_service_and_completed_construction_not_classified_electric,plant_in_service_ferc1,transmission_plant,electric,,,1.0 -retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings,total,,,1.0 -retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,retained_earnings_ferc1,appropriated_retained_earnings_amortization_reserve_federal,total,,,1.0 -retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,appropriated_retained_earnings_including_reserve_amortization,total,,,1.0 -retained_earnings_ferc1,retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_credit,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,adjustments_to_retained_earnings_debit,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,appropriations_of_retained_earnings,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,balance_transferred_from_income,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_common_stock,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,dividends_declared_preferred_stock,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,transfers_from_unappropriated_undistributed_subsidiary_earnings,total,,,1.0 -retained_earnings_ferc1,unappropriated_retained_earnings,retained_earnings_ferc1,unappropriated_retained_earnings_previous_year,total,,,1.0 -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,changes_unappropriated_undistributed_subsidiary_earnings_credits,total,,,1.0 -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,dividends_received,total,,,-1.0 -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,equity_in_earnings_of_subsidiary_companies,total,,,1.0 -retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings,retained_earnings_ferc1,unappropriated_undistributed_subsidiary_earnings_previous_year,total,,,1.0 -utility_plant_summary_ferc1,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,utility_plant_summary_ferc1,depreciation_amortization_and_depletion_utility_plant_in_service,,,, -utility_plant_summary_ferc1,depreciation_utility_plant_in_service,electric_plant_depreciation_functional_ferc1,accumulated_depreciation,electric,in_service,total,1.0 -utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified,,,, -utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_classified_and_property_under_capital_leases,,,,1.0 -utility_plant_summary_ferc1,utility_plant_in_service_classified_and_unclassified,utility_plant_summary_ferc1,utility_plant_in_service_property_under_capital_leases,,,, -utility_plant_summary_ferc1,utility_plant_in_service_experimental_plant_unclassified,plant_in_service_ferc1,experimental_electric_plant_unclassified,electric,in_service,,1.0 -utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_purchased,electric,,,1.0 -utility_plant_summary_ferc1,utility_plant_in_service_plant_purchased_or_sold,plant_in_service_ferc1,electric_plant_sold,electric,,,-1.0 -utility_plant_summary_ferc1,utility_plant_and_construction_work_in_progress,balance_sheet_assets_ferc1,utility_plant,,,, +table_name_parent,xbrl_factoid_parent,table_name,xbrl_factoid,weight,utility_type,plant_function,plant_status +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,abandonment_of_leases,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,1.0,total,, +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,amortization_of_other_utility_plant_utility_plant_in_service,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,amortization_of_plant_acquisition_adjustment,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,depreciation_amortization_and_depletion_utility_plant_in_service,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,depreciation_amortization_and_depletion_utility_plant_leased_to_others,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,depreciation_and_amortization_utility_plant_held_for_future_use,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,depreciation_utility_plant_in_service,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,construction_work_in_progress,core_ferc1__yearly_utility_plant_summary_sched200,construction_work_in_progress,1.0,total,, +core_ferc1__yearly_balance_sheet_assets_sched110,current_and_accrued_assets,core_ferc1__yearly_balance_sheet_assets_sched110,derivative_instrument_assets_hedges_long_term,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,current_and_accrued_assets,core_ferc1__yearly_balance_sheet_assets_sched110,derivative_instrument_assets_long_term,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,current_and_accrued_assets,core_ferc1__yearly_balance_sheet_assets_sched110,less_derivative_instrument_assets_hedges_long_term,-1.0,,, +core_ferc1__yearly_balance_sheet_assets_sched110,current_and_accrued_assets,core_ferc1__yearly_balance_sheet_assets_sched110,less_derivative_instrument_assets_long_term,-1.0,,, +core_ferc1__yearly_balance_sheet_assets_sched110,current_and_accrued_assets,core_ferc1__yearly_balance_sheet_assets_sched110,less_noncurrent_portion_of_allowances,-1.0,,, +core_ferc1__yearly_balance_sheet_assets_sched110,current_and_accrued_assets,core_ferc1__yearly_balance_sheet_assets_sched110,noncurrent_portion_of_allowances,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,deferred_debits,core_ferc1__yearly_balance_sheet_assets_sched110,preliminary_natural_gas_survey_and_investigation_charges,1.0,,, +core_ferc1__yearly_balance_sheet_assets_sched110,deferred_debits,core_ferc1__yearly_balance_sheet_assets_sched110,preliminary_natural_gas_and_other_survey_and_investigation_charges,1.0,,, +core_ferc1__yearly_balance_sheet_assets_sched110,nuclear_fuel_net,core_ferc1__yearly_balance_sheet_assets_sched110,nuclear_fuel,1.0,,, +core_ferc1__yearly_balance_sheet_assets_sched110,nuclear_fuel_net,nuclear_fuel_materials_ferc1,nuclear_fuel_materials_and_assemblies,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,nuclear_fuel_net,nuclear_fuel_materials_ferc1,spent_nuclear_fuel,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,other_property_and_investments,core_ferc1__yearly_balance_sheet_assets_sched110,special_funds_all,1.0,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_balance_sheet_assets_sched110,construction_work_in_progress,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_acquisition_adjustment,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_and_construction_work_in_progress,1.0,total,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_held_for_future_use,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_classified_and_unclassified,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_leased_to_others,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_net,core_ferc1__yearly_balance_sheet_assets_sched110,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_net,core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_and_construction_work_in_progress,,,, +core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant_net,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_net,1.0,total,, +core_ferc1__yearly_balance_sheet_liabilities_sched110,deferred_credits,core_ferc1__yearly_balance_sheet_liabilities_sched110,accumulated_deferred_income_taxes,1.0,,, +core_ferc1__yearly_balance_sheet_liabilities_sched110,retained_earnings,core_ferc1__yearly_retained_earnings_sched118,retained_earnings,1.0,,, +core_ferc1__yearly_balance_sheet_liabilities_sched110,current_and_accrued_liabilities,core_ferc1__yearly_balance_sheet_liabilities_sched110,long_term_portion_of_derivative_instrument_liabilities,,,, +core_ferc1__yearly_balance_sheet_liabilities_sched110,current_and_accrued_liabilities,core_ferc1__yearly_balance_sheet_liabilities_sched110,less_long_term_portion_of_derivative_instrument_liabilities,-1.0,,, +core_ferc1__yearly_balance_sheet_liabilities_sched110,current_and_accrued_liabilities,core_ferc1__yearly_balance_sheet_liabilities_sched110,long_term_portion_of_derivative_instrument_liabilities_hedges,,,, +core_ferc1__yearly_balance_sheet_liabilities_sched110,current_and_accrued_liabilities,core_ferc1__yearly_balance_sheet_liabilities_sched110,less_long_term_portion_of_derivative_instrument_liabilities_hedges,-1.0,,, +core_ferc1__yearly_energy_sources_sched401,sources_of_energy,core_ferc1__yearly_energy_sources_sched401,megawatt_hours_purchased,1.0,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,electric_expenses_hydraulic_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,hydraulic_expenses,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_electric_plant_hydraulic_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_miscellaneous_hydraulic_plant,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_reservoirs_dams_and_waterways,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_structures_hydraulic_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_supervision_and_engineering_hydraulic_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,miscellaneous_hydraulic_power_generation_expenses,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,rents_hydraulic_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_operating_expenses_sched320,water_for_power,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_hydroelectric_plants_sched406,opex_operations,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_pumped_storage_plants_sched408,opex_operations,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_hydraulic_power,core_ferc1__yearly_steam_plants_sched402,opex_operations,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,allowances,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,coolants_and_water,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,electric_expenses_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,fuel_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_boiler_plant_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_electric_plant_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_miscellaneous_steam_plant,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_structures_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,maintenance_supervision_and_engineering_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,miscellaneous_steam_power_expenses,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,rents_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,steam_expenses_steam_power_generation,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,steam_from_other_sources,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_operating_expenses_sched320,steam_transferred_credit,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_hydroelectric_plants_sched406,opex_operations,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_pumped_storage_plants_sched408,opex_operations,,,, +core_ferc1__yearly_operating_expenses_sched320,power_production_expenses_steam_power,core_ferc1__yearly_steam_plants_sched402,opex_operations,,,, +core_ferc1__yearly_operating_expenses_sched320,transmission_operation_expense,core_ferc1__yearly_operating_expenses_sched320,load_dispatching_transmission_expense,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,forfeited_discounts,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,interdepartmental_rents,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,miscellaneous_revenue,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,miscellaneous_service_revenues,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,other_electric_revenue,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,other_miscellaneous_operating_revenues,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,regional_transmission_service_revenues,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,rent_from_electric_property,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,revenues_from_transmission_of_electricity_of_others,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,other_operating_revenues,core_ferc1__yearly_operating_revenues_sched300,sales_of_water_and_water_power,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,sales_to_ultimate_consumers,core_ferc1__yearly_operating_revenues_sched300,large_or_industrial,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,sales_to_ultimate_consumers,core_ferc1__yearly_operating_revenues_sched300,small_or_commercial,1.0,,, +core_ferc1__yearly_operating_revenues_sched300,sales_to_ultimate_consumers,core_ferc1__yearly_sales_by_rate_schedules_sched304,commercial_and_industrial,,,, +core_ferc1__yearly_depreciation_changes_sched219,ending_balance,core_ferc1__yearly_depreciation_changes_sched219,book_cost_of_asset_retirement_costs,1.0,,, +core_ferc1__yearly_depreciation_changes_sched219,ending_balance,core_ferc1__yearly_depreciation_changes_sched219,depreciation_provision,1.0,,, +core_ferc1__yearly_depreciation_changes_sched219,ending_balance,core_ferc1__yearly_depreciation_changes_sched219,net_charges_for_retired_plant,1.0,,, +core_ferc1__yearly_depreciation_changes_sched219,ending_balance,core_ferc1__yearly_depreciation_changes_sched219,other_adjustments_to_accumulated_depreciation,1.0,,, +core_ferc1__yearly_depreciation_changes_sched219,ending_balance,core_ferc1__yearly_depreciation_changes_sched219,starting_balance,1.0,,, +core_ferc1__yearly_income_statements_sched114,amortization_and_depletion_of_utility_plant,core_ferc1__yearly_depreciation_summary_sched336,amortization_limited_term_electric_plant,1.0,electric,total, +core_ferc1__yearly_income_statements_sched114,amortization_and_depletion_of_utility_plant,core_ferc1__yearly_depreciation_summary_sched336,amortization_other_electric_plant,1.0,electric,total, +core_ferc1__yearly_income_statements_sched114,depreciation_expense,core_ferc1__yearly_depreciation_summary_sched336,depreciation_expense,1.0,electric,total, +core_ferc1__yearly_income_statements_sched114,depreciation_expense_for_asset_retirement_costs,core_ferc1__yearly_depreciation_summary_sched336,depreciation_expense_asset_retirement,1.0,electric,total, +core_ferc1__yearly_income_statements_sched114,income_before_extraordinary_items,core_ferc1__yearly_income_statements_sched114,net_utility_operating_income,1.0,,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,distribution_maintenance_expense_electric,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,hydraulic_power_generation_maintenance_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,maintenance_of_general_plant,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,nuclear_power_generation_maintenance_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,other_power_generation_maintenance_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,regional_market_maintenance_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,steam_power_generation_maintenance_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,maintenance_expense,core_ferc1__yearly_operating_expenses_sched320,transmission_maintenance_expense_electric,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operating_revenues,core_ferc1__yearly_operating_revenues_sched300,electric_operating_revenues,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,administrative_and_general_operation_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,customer_account_expenses,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,customer_service_and_information_expenses,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,distribution_operation_expenses_electric,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,hydraulic_power_generation_operations_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,nuclear_power_generation_operations_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,other_power_generation_operations_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,regional_market_operation_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,sales_expenses,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,steam_power_generation_operations_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,transmission_operation_expense,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,operation_expense,core_ferc1__yearly_operating_expenses_sched320,power_production_expenses,1.0,electric,, +core_ferc1__yearly_income_statements_sched114,other_income_deductions,core_ferc1__yearly_income_statements_sched114,miscellaneous_deductions,1.0,,, +core_ferc1__yearly_income_statements_sched114,taxes_on_other_income_and_deductions,core_ferc1__yearly_income_statements_sched114,investment_tax_credits,-1.0,,, +core_ferc1__yearly_plant_in_service_sched204,electric_plant_in_service_and_completed_construction_not_classified_electric,core_ferc1__yearly_plant_in_service_sched204,distribution_plant,1.0,,, +core_ferc1__yearly_plant_in_service_sched204,electric_plant_in_service_and_completed_construction_not_classified_electric,core_ferc1__yearly_plant_in_service_sched204,general_plant,1.0,,, +core_ferc1__yearly_plant_in_service_sched204,electric_plant_in_service_and_completed_construction_not_classified_electric,core_ferc1__yearly_plant_in_service_sched204,intangible_plant,1.0,,, +core_ferc1__yearly_plant_in_service_sched204,electric_plant_in_service_and_completed_construction_not_classified_electric,core_ferc1__yearly_plant_in_service_sched204,production_plant,1.0,,, +core_ferc1__yearly_plant_in_service_sched204,electric_plant_in_service_and_completed_construction_not_classified_electric,core_ferc1__yearly_plant_in_service_sched204,transmission_and_market_operation_plant_regional_transmission_and_market_operation_plant,1.0,,, +core_ferc1__yearly_plant_in_service_sched204,electric_plant_in_service_and_completed_construction_not_classified_electric,core_ferc1__yearly_plant_in_service_sched204,transmission_plant,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,appropriated_retained_earnings_including_reserve_amortization,core_ferc1__yearly_retained_earnings_sched118,appropriated_retained_earnings,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,appropriated_retained_earnings_including_reserve_amortization,core_ferc1__yearly_retained_earnings_sched118,appropriated_retained_earnings_amortization_reserve_federal,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,retained_earnings,core_ferc1__yearly_retained_earnings_sched118,appropriated_retained_earnings_including_reserve_amortization,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,retained_earnings,core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,adjustments_to_retained_earnings_credit,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,adjustments_to_retained_earnings_debit,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,appropriations_of_retained_earnings,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,balance_transferred_from_income,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,dividends_declared_common_stock,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,dividends_declared_preferred_stock,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,transfers_from_unappropriated_undistributed_subsidiary_earnings,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings,core_ferc1__yearly_retained_earnings_sched118,unappropriated_retained_earnings_previous_year,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_undistributed_subsidiary_earnings,core_ferc1__yearly_retained_earnings_sched118,changes_unappropriated_undistributed_subsidiary_earnings_credits,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_undistributed_subsidiary_earnings,core_ferc1__yearly_retained_earnings_sched118,dividends_received,-1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_undistributed_subsidiary_earnings,core_ferc1__yearly_retained_earnings_sched118,equity_in_earnings_of_subsidiary_companies,1.0,,, +core_ferc1__yearly_retained_earnings_sched118,unappropriated_undistributed_subsidiary_earnings,core_ferc1__yearly_retained_earnings_sched118,unappropriated_undistributed_subsidiary_earnings_previous_year,1.0,,, +core_ferc1__yearly_balance_sheet_liabilities_sched110,unappropriated_undistributed_subsidiary_earnings,core_ferc1__yearly_retained_earnings_sched118,unappropriated_undistributed_subsidiary_earnings,1.0,,, +core_ferc1__yearly_utility_plant_summary_sched200,accumulated_provision_for_depreciation_amortization_and_depletion_of_plant_utility,core_ferc1__yearly_utility_plant_summary_sched200,depreciation_amortization_and_depletion_utility_plant_in_service,,,, +core_ferc1__yearly_utility_plant_summary_sched200,depreciation_utility_plant_in_service,core_ferc1__yearly_depreciation_by_function_sched219,accumulated_depreciation,1.0,electric,total,in_service +core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_classified_and_unclassified,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_classified,,,, +core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_classified_and_unclassified,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_classified_and_property_under_capital_leases,1.0,,, +core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_classified_and_unclassified,core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_property_under_capital_leases,,,, +core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_experimental_plant_unclassified,core_ferc1__yearly_plant_in_service_sched204,experimental_electric_plant_unclassified,1.0,electric,, +core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_plant_purchased_or_sold,core_ferc1__yearly_plant_in_service_sched204,electric_plant_purchased,1.0,electric,, +core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_in_service_plant_purchased_or_sold,core_ferc1__yearly_plant_in_service_sched204,electric_plant_sold,-1.0,electric,, +core_ferc1__yearly_utility_plant_summary_sched200,utility_plant_and_construction_work_in_progress,core_ferc1__yearly_balance_sheet_assets_sched110,utility_plant,,,, diff --git a/src/pudl/package_data/glue/ferc1_eia_null.csv b/src/pudl/package_data/glue/eia_ferc1_null.csv similarity index 100% rename from src/pudl/package_data/glue/ferc1_eia_null.csv rename to src/pudl/package_data/glue/eia_ferc1_null.csv diff --git a/src/pudl/package_data/glue/ferc1_eia_one_to_many.csv b/src/pudl/package_data/glue/eia_ferc1_one_to_many.csv similarity index 100% rename from src/pudl/package_data/glue/ferc1_eia_one_to_many.csv rename to src/pudl/package_data/glue/eia_ferc1_one_to_many.csv diff --git a/src/pudl/package_data/glue/ferc1_eia_one_to_many_orig.csv b/src/pudl/package_data/glue/eia_ferc1_one_to_many_orig.csv similarity index 100% rename from src/pudl/package_data/glue/ferc1_eia_one_to_many_orig.csv rename to src/pudl/package_data/glue/eia_ferc1_one_to_many_orig.csv diff --git a/src/pudl/package_data/glue/ferc1_eia_train.csv b/src/pudl/package_data/glue/eia_ferc1_train.csv similarity index 100% rename from src/pudl/package_data/glue/ferc1_eia_train.csv rename to src/pudl/package_data/glue/eia_ferc1_train.csv diff --git a/src/pudl/transform/classes.py b/src/pudl/transform/classes.py index 0f41e609b4..9d4a0e50c3 100644 --- a/src/pudl/transform/classes.py +++ b/src/pudl/transform/classes.py @@ -1092,7 +1092,7 @@ class AbstractTableTransformer(ABC): single dataframe. Since Python is lazy about enforcing types and interfaces you can get away with other kinds of arguments when they're sometimes necessary, but this isn't a good arrangement and we should figure out how to do it right. See the - :class:`pudl.transform.ferc1.PlantsSteamFerc1TableTransformer` class for an example. + :class:`pudl.transform.ferc1.SteamPlantsTableTransformer` class for an example. """ table_id: enum.Enum @@ -1181,7 +1181,7 @@ def transform_main(self, df: pd.DataFrame, **kwargs) -> pd.DataFrame: return a single dataframe, and that pattern is implemented in the :meth:`AbstractTableTransformer.transform` method. In cases where transforms take or return more than one dataframe, you will need to define a new transform - method within the child class. See :class:`PlantsSteamFerc1TableTransformer` + method within the child class. See :class:`SteamPlantsTableTransformer` as an example. """ ... diff --git a/src/pudl/transform/eia.py b/src/pudl/transform/eia.py index c2ee33a516..d2d1a360ba 100644 --- a/src/pudl/transform/eia.py +++ b/src/pudl/transform/eia.py @@ -358,7 +358,7 @@ def _compile_all_entity_records( # encode the compiled options! compiled_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource(f"{entity.value}_eia860") + .get_resource(f"core_eia860__scd_{entity.value}") .encode(compiled_df) ) return compiled_df @@ -583,9 +583,9 @@ def harvest_entity_tables( # noqa: C901 # Apply standard PUDL data types to the new entity tables: pkg = Package.from_resource_ids() - entity_res = pkg.get_resource(f"{entity.value}_entity_eia") + entity_res = pkg.get_resource(f"core_eia__entity_{entity.value}") entity_df = apply_pudl_dtypes(entity_df, group="eia").pipe(entity_res.encode) - annual_res = pkg.get_resource(f"{entity.value}_eia860") + annual_res = pkg.get_resource(f"core_eia860__scd_{entity.value}") annual_df = apply_pudl_dtypes(annual_df, group="eia").pipe(annual_res.encode) if entity == EiaEntity.PLANTS: @@ -623,7 +623,7 @@ def harvest_entity_tables( # noqa: C901 required_resource_keys={"dataset_settings"}, io_manager_key="pudl_sqlite_io_manager", ) -def boiler_generator_assn_eia860(context, **clean_dfs) -> pd.DataFrame: # noqa: C901 +def core_eia860__assn_boiler_generator(context, **clean_dfs) -> pd.DataFrame: # noqa: C901 """Creates a set of more complete boiler generator associations. Creates a unique unit_id_pudl for each collection of boilers and generators @@ -674,7 +674,7 @@ def boiler_generator_assn_eia860(context, **clean_dfs) -> pd.DataFrame: # noqa: logger.info("Inferring complete EIA boiler-generator associations.") logger.debug(f"{clean_dfs.keys()=}") - # grab the generation_eia923 table, group annually, generate a new tag + # grab the core_eia923__monthly_generation table, group annually, generate a new tag gen_eia923 = clean_dfs["_core_eia923__generation"] gen_eia923 = ( gen_eia923.set_index(pd.DatetimeIndex(gen_eia923.report_date)) @@ -1167,15 +1167,15 @@ def harvested_entity_asset_factory( "_core_eia860__utilities", "_core_eia860__emissions_control_equipment", "_core_eia860__boiler_emissions_control_equipment_assn", - "_core_eia860__boiler_cooling_assn", - "_core_eia860__boiler_stack_flue_assn", + "_core_eia860__boiler_cooling", + "_core_eia860__boiler_stack_flue", ) @multi_asset( ins={table_name: AssetIn() for table_name in harvestable_assets}, outs={ - f"{entity.value}_entity_eia": AssetOut(io_manager_key=io_manager_key), - f"{entity.value}_eia860": AssetOut(io_manager_key=io_manager_key), + f"core_eia__entity_{entity.value}": AssetOut(io_manager_key=io_manager_key), + f"core_eia860__scd_{entity.value}": AssetOut(io_manager_key=io_manager_key), }, config_schema={ "debug": Field( @@ -1201,8 +1201,8 @@ def harvested_entity(context, **clean_dfs): ) return ( - Output(output_name=f"{entity.value}_entity_eia", value=entity_df), - Output(output_name=f"{entity.value}_eia860", value=annual_df), + Output(output_name=f"core_eia__entity_{entity.value}", value=entity_df), + Output(output_name=f"core_eia860__scd_{entity.value}", value=annual_df), ) return harvested_entity @@ -1215,15 +1215,18 @@ def harvested_entity(context, **clean_dfs): def finished_eia_asset_factory( - table_name: str, io_manager_key: str | None = None + table_name: str, _core_table_name: str, io_manager_key: str | None = None ) -> AssetsDefinition: - """An asset factory for finished EIA tables.""" - # TODO (bendnorman): Create a more graceful function for parsing table name - table_name_parts = table_name.split("_") - dataset = table_name_parts[-1] - table_name_no_dataset = "_".join(table_name_parts[:-1]) + """An asset factory for finished EIA tables. - _core_table_name = f"_core_{dataset}__{table_name_no_dataset}" + Args: + table_name: the name of the harvest table. + _core_table_name: the name of the unharvested input table + io_manager_key: the name of the IO Manager of the final asset. + + Returns: + A harvest EIA asset. + """ @asset( ins={_core_table_name: AssetIn()}, @@ -1240,18 +1243,20 @@ def finished_eia_asset(**kwargs) -> pd.DataFrame: finished_eia_assets = [ - finished_eia_asset_factory(table_name, io_manager_key="pudl_sqlite_io_manager") - for table_name in [ - "boiler_fuel_eia923", - "coalmine_eia923", - "fuel_receipts_costs_eia923", - "generation_eia923", - "generation_fuel_eia923", - "generation_fuel_nuclear_eia923", - "ownership_eia860", - "emissions_control_equipment_eia860", - "boiler_emissions_control_equipment_assn_eia860", - "boiler_cooling_assn_eia860", - "boiler_stack_flue_assn_eia860", - ] + finished_eia_asset_factory( + table_name, _core_table_name, io_manager_key="pudl_sqlite_io_manager" + ) + for table_name, _core_table_name in { + "core_eia923__monthly_boiler_fuel": "_core_eia923__boiler_fuel", + "core_eia923__entity_coalmine": "_core_eia923__coalmine", + "core_eia923__monthly_fuel_receipts_costs": "_core_eia923__fuel_receipts_costs", + "core_eia923__monthly_generation": "_core_eia923__generation", + "core_eia923__monthly_generation_fuel": "_core_eia923__generation_fuel", + "core_eia923__monthly_generation_fuel_nuclear": "_core_eia923__generation_fuel_nuclear", + "core_eia860__scd_ownership": "_core_eia860__ownership", + "core_eia860__scd_emissions_control_equipment": "_core_eia860__emissions_control_equipment", + "core_eia860__assn_yearly_boiler_emissions_control_equipment": "_core_eia860__boiler_emissions_control_equipment_assn", + "core_eia860__assn_boiler_cooling": "_core_eia860__boiler_cooling", + "core_eia860__assn_boiler_stack_flue": "_core_eia860__boiler_stack_flue", + }.items() ] diff --git a/src/pudl/transform/eia860.py b/src/pudl/transform/eia860.py index d64a8c0b17..13bf75efbb 100644 --- a/src/pudl/transform/eia860.py +++ b/src/pudl/transform/eia860.py @@ -25,10 +25,10 @@ def _core_eia860__ownership(raw_eia860__ownership: pd.DataFrame) -> pd.DataFrame reporting. Args: - raw_eia860__ownership: The raw ``ownership_eia860`` dataframe. + raw_eia860__ownership: The raw ``raw_eia860__ownership`` dataframe. Returns: - Cleaned ``ownership_eia860`` dataframe ready for harvesting. + Cleaned ``_core_eia860__ownership`` dataframe ready for harvesting. """ # Preiminary clean and get rid of unecessary 'year' column own_df = ( @@ -95,7 +95,7 @@ def _core_eia860__ownership(raw_eia860__ownership: pd.DataFrame) -> pd.DataFrame remaining_dupes = own_df[own_df.duplicated(subset=own_pk, keep=False)] if not remaining_dupes.empty: raise ValueError( - f"Duplicate ownership slices found in ownership_eia860: {remaining_dupes}" + f"Duplicate ownership slices found in _core_eia860__ownership: {remaining_dupes}" ) # Remove a couple of records known to have (literal) "nan" values in the @@ -121,7 +121,9 @@ def _core_eia860__ownership(raw_eia860__ownership: pd.DataFrame) -> pd.DataFrame own_df = own_df[~mask] if not (nulls := own_df[own_df.generator_id == ""]).empty: - logger.warning(f"Found records with null IDs in ownership_eia860: {nulls}") + logger.warning( + f"Found records with null IDs in _core_eia860__ownership: {nulls}" + ) # In 2010 there are several hundred utilities that appear to be incorrectly # reporting the owner_utility_id_eia value *also* in the utility_id_eia # column. This results in duplicate operator IDs associated with a given @@ -150,7 +152,7 @@ def _core_eia860__ownership(raw_eia860__ownership: pd.DataFrame) -> pd.DataFrame own_df.loc[single_owner_operator, "operator_utility_id_eia"] = pd.NA own_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("ownership_eia860") + .get_resource("core_eia860__scd_ownership") .encode(own_df) ) # CN is an invalid political subdivision code used by a few respondents to indicate @@ -213,7 +215,7 @@ def _core_eia860__generators( raw_eia860__generator: The raw ``raw_eia860__generator`` dataframe. Returns: - Cleaned ``generators_eia860`` dataframe ready for harvesting. + Cleaned ``_core_eia860__generators`` dataframe ready for harvesting. """ # Groupby objects were creating chained assignment warning that is N/A pd.options.mode.chained_assignment = None @@ -332,13 +334,13 @@ def _core_eia860__generators( gens_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("generators_eia860") + .get_resource("core_eia860__scd_generators") .encode(gens_df) ) gens_df["fuel_type_code_pudl"] = gens_df.energy_source_code_1.str.upper().map( pudl.helpers.label_map( - CODE_METADATA["energy_sources_eia"]["df"], + CODE_METADATA["core_eia__codes_energy_sources"]["df"], from_col="code", to_col="fuel_type_code_pudl", null_value=pd.NA, @@ -347,7 +349,7 @@ def _core_eia860__generators( gens_df["operational_status"] = gens_df.operational_status_code.str.upper().map( pudl.helpers.label_map( - CODE_METADATA["operational_status_eia"]["df"], + CODE_METADATA["core_eia__codes_operational_status"]["df"], from_col="code", to_col="operational_status", null_value=pd.NA, @@ -376,9 +378,9 @@ def _core_eia860__plants(raw_eia860__plant: pd.DataFrame) -> pd.DataFrame: raw_eia860__plant: The raw ``raw_eia860__plant`` dataframe. Returns: - Cleaned ``plants_eia860`` dataframe ready for harvesting. + Cleaned ``_core_eia860__plants`` dataframe ready for harvesting. """ - # Populating the 'plants_eia860' table + # Populating the '_core_eia860__plants' table p_df = ( raw_eia860__plant.pipe(pudl.helpers.fix_eia_na) .astype({"zip_code": str}) @@ -435,7 +437,7 @@ def _core_eia860__plants(raw_eia860__plant: pd.DataFrame) -> pd.DataFrame: p_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("plants_eia860") + .get_resource("core_eia860__scd_plants") .encode(p_df) ) @@ -459,9 +461,9 @@ def _core_eia860__boiler_generator_assn( spreadsheets they distribute. Returns: - Cleaned ``boiler_generator_assn_eia860`` dataframe ready for harvesting. + Cleaned ``_core_eia860__boiler_generator_assn`` dataframe ready for harvesting. """ - # Populating the 'generators_eia860' table + # Populating the 'core_eia860__scd_generators' table b_g_df = raw_eia860__boiler_generator_assn b_g_df = pudl.helpers.convert_to_date(b_g_df) @@ -470,7 +472,7 @@ def _core_eia860__boiler_generator_assn( b_g_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("boiler_generator_assn_eia860") + .get_resource("core_eia860__assn_boiler_generator") .encode(b_g_df) ) @@ -495,9 +497,9 @@ def _core_eia860__utilities(raw_eia860__utility: pd.DataFrame) -> pd.DataFrame: raw_eia860__utility: The raw ``raw_eia860__utility`` dataframe. Returns: - Cleaned ``utilities_eia860`` dataframe ready for harvesting. + Cleaned ``_core_eia860__utilities`` dataframe ready for harvesting. """ - # Populating the 'utilities_eia860' table + # Populating the '_core_eia860__utilities' table u_df = raw_eia860__utility # Replace empty strings, whitespace, and '.' fields with real NA values @@ -585,7 +587,7 @@ def _core_eia860__boilers( Returns: pandas.DataFrame: the transformed boilers table """ - # Populating the 'boilers_eia860' table + # Populating the 'core_eia860__scd_boilers' table b_df = raw_eia860__boiler_info ecs = raw_eia860__emission_control_strategies @@ -728,7 +730,9 @@ def _core_eia860__boilers( # Add boiler manufacturer name to column b_df["boiler_manufacturer"] = b_df.boiler_manufacturer_code.map( pudl.helpers.label_map( - CODE_METADATA["environmental_equipment_manufacturers_eia"]["df"], + CODE_METADATA["core_eia__codes_environmental_equipment_manufacturers"][ + "df" + ], from_col="code", to_col="description", null_value=pd.NA, @@ -737,7 +741,9 @@ def _core_eia860__boilers( b_df["nox_control_manufacturer"] = b_df.nox_control_manufacturer_code.map( pudl.helpers.label_map( - CODE_METADATA["environmental_equipment_manufacturers_eia"]["df"], + CODE_METADATA["core_eia__codes_environmental_equipment_manufacturers"][ + "df" + ], from_col="code", to_col="description", null_value=pd.NA, @@ -755,7 +761,7 @@ def _core_eia860__boilers( b_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("boilers_eia860") + .get_resource("core_eia860__scd_boilers") .encode(b_df) ) @@ -879,7 +885,7 @@ def _core_eia860__emissions_control_equipment( emce_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("emissions_control_equipment_eia860") + .get_resource("core_eia860__scd_emissions_control_equipment") .encode(emce_df) ) @@ -967,7 +973,7 @@ def _core_eia860__boiler_emissions_control_equipment_assn( @asset -def _core_eia860__boiler_cooling_assn( +def _core_eia860__boiler_cooling( raw_eia860__boiler_cooling: pd.DataFrame, ) -> pd.DataFrame: """Pull and transform the EIA 860 boiler to cooler ID table. @@ -992,7 +998,7 @@ def _core_eia860__boiler_cooling_assn( @asset -def _core_eia860__boiler_stack_flue_assn( +def _core_eia860__boiler_stack_flue( raw_eia860__boiler_stack_flue: pd.DataFrame, ) -> pd.DataFrame: """Pull and transform the EIA 860 boiler to stack flue ID table. diff --git a/src/pudl/transform/eia861.py b/src/pudl/transform/eia861.py index 46c4d9bdf7..7c082fe0af 100644 --- a/src/pudl/transform/eia861.py +++ b/src/pudl/transform/eia861.py @@ -620,10 +620,10 @@ def backfill_ba_codes_by_ba_id(df: pd.DataFrame) -> pd.DataFrame: Args: df: The transformed EIA 861 Balancing Authority dataframe - (balancing_authority_eia861). + (core_eia861__yearly_balancing_authority). Returns: - pandas.DataFrame: The balancing_authority_eia861 dataframe, but with many fewer + pandas.DataFrame: The core_eia861__yearly_balancing_authority dataframe, but with many fewer NA values in the balancing_authority_code_eia column. """ ba_eia861_filled = ( @@ -930,7 +930,7 @@ def _compare_nerc_physical_w_nerc_operational(df: pd.DataFrame) -> pd.DataFrame: year where there is a match between the cols. Args: - df: utility_data_nerc_eia861 table from :func:`clean_utility_data_eia861` + df: core_eia861__yearly_utility_data_nerc table. Returns: A DataFrame with rows for utilities where NO listed operating nerc region @@ -1026,7 +1026,7 @@ def _harvest_associations(dfs: list[pd.DataFrame], cols: list[str]) -> pd.DataFr # EIA Form 861 Table Transform Functions ############################################################################### @asset(io_manager_key="pudl_sqlite_io_manager") -def service_territory_eia861( +def core_eia861__yearly_service_territory( raw_eia861__service_territory: pd.DataFrame, ) -> pd.DataFrame: """Transform the EIA 861 utility service territory table. @@ -1067,7 +1067,7 @@ def service_territory_eia861( pk = ( Package.from_resource_ids() - .get_resource("service_territory_eia861") + .get_resource("core_eia861__yearly_service_territory") .schema.primary_key ) # We've fixed all we can fix! ~99.84% FIPS coverage. @@ -1080,7 +1080,7 @@ def service_territory_eia861( @asset -def clean_balancing_authority_eia861( +def _core_eia861__balancing_authority( raw_eia861__balancing_authority: pd.DataFrame, ) -> pd.DataFrame: """Transform the EIA 861 Balancing Authority table. @@ -1139,7 +1139,7 @@ def clean_balancing_authority_eia861( @asset(io_manager_key="pudl_sqlite_io_manager") -def sales_eia861(raw_eia861__sales: pd.DataFrame) -> pd.DataFrame: +def core_eia861__yearly_sales(raw_eia861__sales: pd.DataFrame) -> pd.DataFrame: """Transform the EIA 861 Sales table. Transformations include: @@ -1222,7 +1222,7 @@ def sales_eia861(raw_eia861__sales: pd.DataFrame) -> pd.DataFrame: @asset(io_manager_key="pudl_sqlite_io_manager") -def advanced_metering_infrastructure_eia861( +def core_eia861__yearly_advanced_metering_infrastructure( raw_eia861__advanced_metering_infrastructure: pd.DataFrame, ) -> pd.DataFrame: """Transform the EIA 861 Advanced Metering Infrastructure table. @@ -1267,13 +1267,15 @@ def advanced_metering_infrastructure_eia861( @multi_asset( outs={ - "demand_response_eia861": AssetOut(io_manager_key="pudl_sqlite_io_manager"), - "demand_response_water_heater_eia861": AssetOut( + "core_eia861__yearly_demand_response": AssetOut( + io_manager_key="pudl_sqlite_io_manager" + ), + "core_eia861__yearly_demand_response_water_heater": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), }, ) -def demand_response_eia861(raw_eia861__demand_response: pd.DataFrame): +def core_eia861__yearly_demand_response(raw_eia861__demand_response: pd.DataFrame): """Transform the EIA 861 Demand Response table. Transformations include: @@ -1338,11 +1340,11 @@ def demand_response_eia861(raw_eia861__demand_response: pd.DataFrame): return ( Output( - output_name="demand_response_eia861", + output_name="core_eia861__yearly_demand_response", value=_post_process(transformed_dr), ), Output( - output_name="demand_response_water_heater_eia861", + output_name="core_eia861__yearly_demand_response_water_heater", value=_post_process(dr_water), ), ) @@ -1350,18 +1352,18 @@ def demand_response_eia861(raw_eia861__demand_response: pd.DataFrame): @multi_asset( outs={ - "demand_side_management_sales_eia861": AssetOut( + "core_eia861__yearly_demand_side_management_sales": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), - "demand_side_management_ee_dr_eia861": AssetOut( + "core_eia861__yearly_demand_side_management_ee_dr": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), - "demand_side_management_misc_eia861": AssetOut( + "core_eia861__yearly_demand_side_management_misc": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), }, ) -def demand_side_management_eia861( +def core_demand_side_management_eia861( raw_eia861__demand_side_management: pd.DataFrame, ): """Transform the EIA 861 Demand Side Management table. @@ -1500,15 +1502,15 @@ def demand_side_management_eia861( return ( Output( - output_name="demand_side_management_sales_eia861", + output_name="core_eia861__yearly_demand_side_management_sales", value=_post_process(dsm_sales), ), Output( - output_name="demand_side_management_ee_dr_eia861", + output_name="core_eia861__yearly_demand_side_management_ee_dr", value=_post_process(dsm_ee_dr), ), Output( - output_name="demand_side_management_misc_eia861", + output_name="core_eia861__yearly_demand_side_management_misc", value=_post_process(dsm_misc), ), ) @@ -1516,18 +1518,18 @@ def demand_side_management_eia861( @multi_asset( outs={ - "distributed_generation_tech_eia861": AssetOut( + "core_eia861__yearly_distributed_generation_tech": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), - "distributed_generation_fuel_eia861": AssetOut( + "core_eia861__yearly_distributed_generation_fuel": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), - "distributed_generation_misc_eia861": AssetOut( + "core_eia861__yearly_distributed_generation_misc": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), }, ) -def distributed_generation_eia861( +def core_distributed_generation_eia861( raw_eia861__distributed_generation: pd.DataFrame, ): """Transform the EIA 861 Distributed Generation table. @@ -1691,22 +1693,22 @@ def distributed_generation_eia861( return ( Output( - output_name="distributed_generation_tech_eia861", + output_name="core_eia861__yearly_distributed_generation_tech", value=_post_process(tidy_dg_tech), ), Output( - output_name="distributed_generation_fuel_eia861", + output_name="core_eia861__yearly_distributed_generation_fuel", value=_post_process(tidy_dg_fuel), ), Output( - output_name="distributed_generation_misc_eia861", + output_name="core_eia861__yearly_distributed_generation_misc", value=_post_process(dg_misc), ), ) @asset(io_manager_key="pudl_sqlite_io_manager") -def distribution_systems_eia861( +def core_eia861__yearly_distribution_systems( raw_eia861__distribution_systems: pd.DataFrame, ) -> pd.DataFrame: """Transform the EIA 861 Distribution Systems table. @@ -1728,7 +1730,9 @@ def distribution_systems_eia861( @asset(io_manager_key="pudl_sqlite_io_manager") -def dynamic_pricing_eia861(raw_eia861__dynamic_pricing: pd.DataFrame) -> pd.DataFrame: +def core_eia861__yearly_dynamic_pricing( + raw_eia861__dynamic_pricing: pd.DataFrame, +) -> pd.DataFrame: """Transform the EIA 861 Dynamic Pricing table. Transformations include: @@ -1790,7 +1794,7 @@ def dynamic_pricing_eia861(raw_eia861__dynamic_pricing: pd.DataFrame) -> pd.Data @asset(io_manager_key="pudl_sqlite_io_manager") -def energy_efficiency_eia861( +def core_eia861__yearly_energy_efficiency( raw_eia861__energy_efficiency: pd.DataFrame, ) -> pd.DataFrame: """Transform the EIA 861 Energy Efficiency table. @@ -1857,7 +1861,7 @@ def energy_efficiency_eia861( @asset(io_manager_key="pudl_sqlite_io_manager") -def green_pricing_eia861( +def core_eia861__yearly_green_pricing( raw_eia861__green_pricing: pd.DataFrame, ) -> pd.DataFrame: """Transform the EIA 861 Green Pricing table. @@ -1901,7 +1905,7 @@ def green_pricing_eia861( @asset(io_manager_key="pudl_sqlite_io_manager") -def mergers_eia861(raw_eia861__mergers: pd.DataFrame) -> pd.DataFrame: +def core_eia861__yearly_mergers(raw_eia861__mergers: pd.DataFrame) -> pd.DataFrame: """Transform the EIA 861 Mergers table.""" # No duplicates to speak of but take measures to check just in case df = ( @@ -1918,13 +1922,15 @@ def mergers_eia861(raw_eia861__mergers: pd.DataFrame) -> pd.DataFrame: @multi_asset( outs={ - "net_metering_customer_fuel_class_eia861": AssetOut( + "core_eia861__yearly_net_metering_customer_fuel_class": AssetOut( + io_manager_key="pudl_sqlite_io_manager" + ), + "core_eia861__yearly_net_metering_misc": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), - "net_metering_misc_eia861": AssetOut(io_manager_key="pudl_sqlite_io_manager"), }, ) -def net_metering_eia861(raw_eia861__net_metering: pd.DataFrame): +def core_net_metering_eia861(raw_eia861__net_metering: pd.DataFrame): """Transform the EIA 861 Net Metering table. Transformations include: @@ -1990,11 +1996,11 @@ def net_metering_eia861(raw_eia861__net_metering: pd.DataFrame): return ( Output( - output_name="net_metering_customer_fuel_class_eia861", + output_name="core_eia861__yearly_net_metering_customer_fuel_class", value=_post_process(tidy_nm_customer_fuel_class), ), Output( - output_name="net_metering_misc_eia861", + output_name="core_eia861__yearly_net_metering_misc", value=_post_process(raw_nm_misc), ), ) @@ -2002,15 +2008,15 @@ def net_metering_eia861(raw_eia861__net_metering: pd.DataFrame): @multi_asset( outs={ - "non_net_metering_customer_fuel_class_eia861": AssetOut( + "core_eia861__yearly_non_net_metering_customer_fuel_class": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), - "non_net_metering_misc_eia861": AssetOut( + "core_eia861__yearly_non_net_metering_misc": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), }, ) -def non_net_metering_eia861(raw_eia861__non_net_metering: pd.DataFrame): +def core_non_net_metering_eia861(raw_eia861__non_net_metering: pd.DataFrame): """Transform the EIA 861 Non-Net Metering table. Transformations include: @@ -2100,11 +2106,11 @@ def non_net_metering_eia861(raw_eia861__non_net_metering: pd.DataFrame): return ( Output( - output_name="non_net_metering_customer_fuel_class_eia861", + output_name="core_eia861__yearly_non_net_metering_customer_fuel_class", value=_post_process(tidy_nnm_customer_fuel_class), ), Output( - output_name="non_net_metering_misc_eia861", + output_name="core_eia861__yearly_non_net_metering_misc", value=_post_process(raw_nnm_misc), ), ) @@ -2112,15 +2118,15 @@ def non_net_metering_eia861(raw_eia861__non_net_metering: pd.DataFrame): @multi_asset( outs={ - "operational_data_revenue_eia861": AssetOut( + "core_eia861__yearly_operational_data_revenue": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), - "operational_data_misc_eia861": AssetOut( + "core_eia861__yearly_operational_data_misc": AssetOut( io_manager_key="pudl_sqlite_io_manager" ), }, ) -def operational_data_eia861(raw_eia861__operational_data: pd.DataFrame): +def core_operational_data_eia861(raw_eia861__operational_data: pd.DataFrame): """Transform the EIA 861 Operational Data table. Transformations include: @@ -2192,18 +2198,20 @@ def operational_data_eia861(raw_eia861__operational_data: pd.DataFrame): return ( Output( - output_name="operational_data_revenue_eia861", + output_name="core_eia861__yearly_operational_data_revenue", value=_post_process(transformed_od_rev), ), Output( - output_name="operational_data_misc_eia861", + output_name="core_eia861__yearly_operational_data_misc", value=_post_process(transformed_od_misc), ), ) @asset(io_manager_key="pudl_sqlite_io_manager") -def reliability_eia861(raw_eia861__reliability: pd.DataFrame) -> pd.DataFrame: +def core_eia861__yearly_reliability( + raw_eia861__reliability: pd.DataFrame, +) -> pd.DataFrame: """Transform the EIA 861 Reliability table. Transformations include: @@ -2260,7 +2268,7 @@ def reliability_eia861(raw_eia861__reliability: pd.DataFrame) -> pd.DataFrame: transformed_r = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("reliability_eia861") + .get_resource("core_eia861__yearly_reliability") .encode(transformed_r) ) @@ -2269,12 +2277,18 @@ def reliability_eia861(raw_eia861__reliability: pd.DataFrame) -> pd.DataFrame: @multi_asset( outs={ - "utility_data_nerc_eia861": AssetOut(io_manager_key="pudl_sqlite_io_manager"), - "utility_data_rto_eia861": AssetOut(io_manager_key="pudl_sqlite_io_manager"), - "utility_data_misc_eia861": AssetOut(io_manager_key="pudl_sqlite_io_manager"), + "core_eia861__yearly_utility_data_nerc": AssetOut( + io_manager_key="pudl_sqlite_io_manager" + ), + "core_eia861__yearly_utility_data_rto": AssetOut( + io_manager_key="pudl_sqlite_io_manager" + ), + "core_eia861__yearly_utility_data_misc": AssetOut( + io_manager_key="pudl_sqlite_io_manager" + ), }, ) -def utility_data_eia861(raw_eia861__utility_data: pd.DataFrame): +def core_utility_data_eia861(raw_eia861__utility_data: pd.DataFrame): """Transform the EIA 861 Utility Data table. Transformations include: @@ -2395,15 +2409,15 @@ def utility_data_eia861(raw_eia861__utility_data: pd.DataFrame): return ( Output( - output_name="utility_data_nerc_eia861", + output_name="core_eia861__yearly_utility_data_nerc", value=_post_process(transformed_ud_nerc), ), Output( - output_name="utility_data_rto_eia861", + output_name="core_eia861__yearly_utility_data_rto", value=_post_process(transformed_ud_rto), ), Output( - output_name="utility_data_misc_eia861", + output_name="core_eia861__yearly_utility_data_misc", value=_post_process(transformed_ud_misc), ), ) @@ -2414,35 +2428,35 @@ def utility_data_eia861(raw_eia861__utility_data: pd.DataFrame): ############################################################################## @asset( ins={ - "advanced_metering_infrastructure_eia861": AssetIn(), - "demand_response_eia861": AssetIn(), - "demand_response_water_heater_eia861": AssetIn(), - "demand_side_management_ee_dr_eia861": AssetIn(), - "demand_side_management_misc_eia861": AssetIn(), - "demand_side_management_sales_eia861": AssetIn(), - "distributed_generation_fuel_eia861": AssetIn(), - "distributed_generation_misc_eia861": AssetIn(), - "distributed_generation_tech_eia861": AssetIn(), - "distribution_systems_eia861": AssetIn(), - "dynamic_pricing_eia861": AssetIn(), - "energy_efficiency_eia861": AssetIn(), - "green_pricing_eia861": AssetIn(), - "mergers_eia861": AssetIn(), - "net_metering_customer_fuel_class_eia861": AssetIn(), - "net_metering_misc_eia861": AssetIn(), - "non_net_metering_customer_fuel_class_eia861": AssetIn(), - "non_net_metering_misc_eia861": AssetIn(), - "operational_data_misc_eia861": AssetIn(), - "operational_data_revenue_eia861": AssetIn(), - "reliability_eia861": AssetIn(), - "sales_eia861": AssetIn(), - "utility_data_misc_eia861": AssetIn(), - "utility_data_nerc_eia861": AssetIn(), - "utility_data_rto_eia861": AssetIn(), + "core_eia861__yearly_advanced_metering_infrastructure": AssetIn(), + "core_eia861__yearly_demand_response": AssetIn(), + "core_eia861__yearly_demand_response_water_heater": AssetIn(), + "core_eia861__yearly_demand_side_management_ee_dr": AssetIn(), + "core_eia861__yearly_demand_side_management_misc": AssetIn(), + "core_eia861__yearly_demand_side_management_sales": AssetIn(), + "core_eia861__yearly_distributed_generation_fuel": AssetIn(), + "core_eia861__yearly_distributed_generation_misc": AssetIn(), + "core_eia861__yearly_distributed_generation_tech": AssetIn(), + "core_eia861__yearly_distribution_systems": AssetIn(), + "core_eia861__yearly_dynamic_pricing": AssetIn(), + "core_eia861__yearly_energy_efficiency": AssetIn(), + "core_eia861__yearly_green_pricing": AssetIn(), + "core_eia861__yearly_mergers": AssetIn(), + "core_eia861__yearly_net_metering_customer_fuel_class": AssetIn(), + "core_eia861__yearly_net_metering_misc": AssetIn(), + "core_eia861__yearly_non_net_metering_customer_fuel_class": AssetIn(), + "core_eia861__yearly_non_net_metering_misc": AssetIn(), + "core_eia861__yearly_operational_data_misc": AssetIn(), + "core_eia861__yearly_operational_data_revenue": AssetIn(), + "core_eia861__yearly_reliability": AssetIn(), + "core_eia861__yearly_sales": AssetIn(), + "core_eia861__yearly_utility_data_misc": AssetIn(), + "core_eia861__yearly_utility_data_nerc": AssetIn(), + "core_eia861__yearly_utility_data_rto": AssetIn(), }, io_manager_key="pudl_sqlite_io_manager", ) -def utility_assn_eia861(**data_dfs: dict[str, pd.DataFrame]) -> pd.DataFrame: +def core_eia861__assn_utility(**data_dfs: dict[str, pd.DataFrame]) -> pd.DataFrame: """Harvest a Utility-Date-State Association Table.""" logger.info("Building an EIA 861 Util-State-Date association table.") df = _harvest_associations( @@ -2453,45 +2467,47 @@ def utility_assn_eia861(**data_dfs: dict[str, pd.DataFrame]) -> pd.DataFrame: @asset( ins={ - "advanced_metering_infrastructure_eia861": AssetIn(), - "clean_balancing_authority_eia861": AssetIn(), - "demand_response_eia861": AssetIn(), - "demand_response_water_heater_eia861": AssetIn(), - "demand_side_management_ee_dr_eia861": AssetIn(), - "demand_side_management_misc_eia861": AssetIn(), - "demand_side_management_sales_eia861": AssetIn(), - "distributed_generation_fuel_eia861": AssetIn(), - "distributed_generation_misc_eia861": AssetIn(), - "distributed_generation_tech_eia861": AssetIn(), - "distribution_systems_eia861": AssetIn(), - "dynamic_pricing_eia861": AssetIn(), - "energy_efficiency_eia861": AssetIn(), - "green_pricing_eia861": AssetIn(), - "mergers_eia861": AssetIn(), - "net_metering_customer_fuel_class_eia861": AssetIn(), - "net_metering_misc_eia861": AssetIn(), - "non_net_metering_customer_fuel_class_eia861": AssetIn(), - "non_net_metering_misc_eia861": AssetIn(), - "operational_data_misc_eia861": AssetIn(), - "operational_data_revenue_eia861": AssetIn(), - "reliability_eia861": AssetIn(), - "sales_eia861": AssetIn(), - "utility_data_misc_eia861": AssetIn(), - "utility_data_nerc_eia861": AssetIn(), - "utility_data_rto_eia861": AssetIn(), + "core_eia861__yearly_advanced_metering_infrastructure": AssetIn(), + "_core_eia861__balancing_authority": AssetIn(), + "core_eia861__yearly_demand_response": AssetIn(), + "core_eia861__yearly_demand_response_water_heater": AssetIn(), + "core_eia861__yearly_demand_side_management_ee_dr": AssetIn(), + "core_eia861__yearly_demand_side_management_misc": AssetIn(), + "core_eia861__yearly_demand_side_management_sales": AssetIn(), + "core_eia861__yearly_distributed_generation_fuel": AssetIn(), + "core_eia861__yearly_distributed_generation_misc": AssetIn(), + "core_eia861__yearly_distributed_generation_tech": AssetIn(), + "core_eia861__yearly_distribution_systems": AssetIn(), + "core_eia861__yearly_dynamic_pricing": AssetIn(), + "core_eia861__yearly_energy_efficiency": AssetIn(), + "core_eia861__yearly_green_pricing": AssetIn(), + "core_eia861__yearly_mergers": AssetIn(), + "core_eia861__yearly_net_metering_customer_fuel_class": AssetIn(), + "core_eia861__yearly_net_metering_misc": AssetIn(), + "core_eia861__yearly_non_net_metering_customer_fuel_class": AssetIn(), + "core_eia861__yearly_non_net_metering_misc": AssetIn(), + "core_eia861__yearly_operational_data_misc": AssetIn(), + "core_eia861__yearly_operational_data_revenue": AssetIn(), + "core_eia861__yearly_reliability": AssetIn(), + "core_eia861__yearly_sales": AssetIn(), + "core_eia861__yearly_utility_data_misc": AssetIn(), + "core_eia861__yearly_utility_data_nerc": AssetIn(), + "core_eia861__yearly_utility_data_rto": AssetIn(), }, io_manager_key="pudl_sqlite_io_manager", ) -def balancing_authority_assn_eia861(**dfs: dict[str, pd.DataFrame]) -> pd.DataFrame: +def core_eia861__assn_balancing_authority( + **dfs: dict[str, pd.DataFrame], +) -> pd.DataFrame: """Compile a balancing authority, utility, state association table. For the years up through 2012, the only BA-Util information that's available comes - from the balancing_authority_eia861 table, and it does not include any state-level + from the core_eia861__yearly_balancing_authority table, and it does not include any state-level information. However, there is utility-state association information in the - sales_eia861 and other data tables. + core_eia861__yearly_sales and other data tables. For the years from 2013 onward, there's explicit BA-Util-State information in the - data tables (e.g. :ref:`sales_eia861`). These observed associations can be compiled + data tables (e.g. :ref:`core_eia861__yearly_sales`). These observed associations can be compiled to give us a picture of which BA-Util-State associations exist. However, we need to merge in the balancing authority IDs since the data tables only contain the balancing authority codes. @@ -2505,10 +2521,10 @@ def balancing_authority_assn_eia861(**dfs: dict[str, pd.DataFrame]) -> pd.DataFr Returns: An association table describing the annual linkages between balancing authorities, states, and utilities. Becomes - :ref:`balancing_authority_assn_eia861`. + :ref:`core_eia861__assn_balancing_authority`. """ # The dataframes from which to compile BA-Util-State associations - ba_eia861 = dfs.pop("clean_balancing_authority_eia861") + ba_eia861 = dfs.pop("_core_eia861__balancing_authority") data_dfs = list(dfs.values()) logger.info("Building an EIA 861 BA-Util-State association table.") @@ -2576,31 +2592,31 @@ def balancing_authority_assn_eia861(**dfs: dict[str, pd.DataFrame]) -> pd.DataFr @asset(io_manager_key="pudl_sqlite_io_manager") -def balancing_authority_eia861( - clean_balancing_authority_eia861: pd.DataFrame, +def core_eia861__yearly_balancing_authority( + _core_eia861__balancing_authority: pd.DataFrame, ) -> pd.DataFrame: - """Finish the normalization of the balancing_authority_eia861 table. + """Finish the normalization of the core_eia861__yearly_balancing_authority table. - The :ref:`balancing_authority_assn_eia861` table depends on information that is only - available in the UN-normalized form of the :ref:`balancing_authority_eia861` table, + The :ref:`core_eia861__assn_balancing_authority` table depends on information that is only + available in the UN-normalized form of the :ref:`core_eia861__yearly_balancing_authority` table, and also on having access to a bunch of transformed data tables, so it can compile the observed combinations of report dates, balancing authorities, states, and utilities. This means that we have to hold off on the final normalization of the - :ref:`balancing_authority_eia861` table until the rest of the transform process is + :ref:`core_eia861__yearly_balancing_authority` table until the rest of the transform process is over. Args: - clean_balancing_authority_eia861: A cleaned up version of the originally + _core_eia861__balancing_authority: A cleaned up version of the originally reported balancing authority table. Returns: - The final, normalized version of the :ref:`balancing_authority_eia861` table, + The final, normalized version of the :ref:`core_eia861__yearly_balancing_authority` table, linking together balancing authorities and utility IDs by year, but without information about what states they were operating in (which is captured in - :ref:`balancing_authority_assn_eia861`). + :ref:`core_eia861__assn_balancing_authority`). """ - logger.info("Completing normalization of balancing_authority_eia861.") - ba_eia861_normed = clean_balancing_authority_eia861.loc[ + logger.info("Completing normalization of core_eia861__yearly_balancing_authority.") + ba_eia861_normed = _core_eia861__balancing_authority.loc[ :, [ "report_date", @@ -2636,7 +2652,7 @@ def balancing_authority_eia861( if delta > 3: logger.warning( "Unexpectedly large number of rows with NULL PK values found in " - f"balancing_authority_eia861. Expected 3, found {delta} (out of " + f"core_eia861__yearly_balancing_authority. Expected 3, found {delta} (out of " f"{len_before} total)." ) diff --git a/src/pudl/transform/eia923.py b/src/pudl/transform/eia923.py index 70dde0d877..28f9ac3cab 100644 --- a/src/pudl/transform/eia923.py +++ b/src/pudl/transform/eia923.py @@ -85,7 +85,7 @@ def _get_plant_nuclear_unit_id_map(nuc_fuel: pd.DataFrame) -> dict[int, str]: def _backfill_nuclear_unit_id(nuc_fuel: pd.DataFrame) -> pd.DataFrame: """Backfill 2001 and 2002 nuclear_unit_id for plants with one nuclear unit. - 2001 and 2002 generation_fuel_eia923 records do not include nuclear_unit_id which is + 2001 and 2002 core_eia923__monthly_generation_fuel records do not include nuclear_unit_id which is required for the primary key of nuclear_unit_fuel_eia923. We backfill this field for plants with one nuclear unit. nuclear_unit_id is filled with 'UNK' if the nuclear_unit_id can't be recovered. @@ -158,7 +158,7 @@ def _get_plant_prime_mover_map(gen_fuel: pd.DataFrame) -> dict[int, str]: def _backfill_prime_mover_code(gen_fuel: pd.DataFrame) -> pd.DataFrame: """Backfill 2001 and 2002 prime_mover_code for plants with one prime mover. - 2001 and 2002 generation_fuel_eia923 records do not include prime_mover_code + 2001 and 2002 core_eia923__monthly_generation_fuel records do not include prime_mover_code which is required for the primary key. We backfill this field for plants with one prime mover. prime_mover_code is set to 'UNK' if future plants have multiple prime movers. @@ -254,7 +254,7 @@ def _clean_gen_fuel_energy_sources(gen_fuel: pd.DataFrame) -> pd.DataFrame: } ) # plant 10204 should be waste heat instead of other. Fixes a mismatch between energy - # source # codes reported in generators_eia860 and the boiler_fuel_eia860 tables. + # source # codes reported in core_eia860__scd_generators and the boiler_fuel_eia860 tables. gen_fuel.loc[ (gen_fuel["plant_id_eia"] == 10204) & (gen_fuel["energy_source_code"] == "OTH"), "energy_source_code", @@ -426,11 +426,11 @@ def _yearly_to_monthly_records(df: pd.DataFrame) -> pd.DataFrame: def _coalmine_cleanup(cmi_df: pd.DataFrame) -> pd.DataFrame: - """Clean up the coalmine_eia923 table. + """Clean up the core_eia923__entity_coalmine table. - This function does most of the coalmine_eia923 table transformation. It is separate + This function does most of the core_eia923__entity_coalmine table transformation. It is separate from the coalmine() transform function because of the peculiar way that we are - normalizing the ref:`fuel_receipts_costs_eia923` table. + normalizing the ref:`core_eia923__monthly_fuel_receipts_costs` table. All of the coalmine information is originally coming from the EIA fuel_receipts_costs spreadsheet, but it really belongs in its own table. We strip it @@ -483,7 +483,7 @@ def _coalmine_cleanup(cmi_df: pd.DataFrame) -> pd.DataFrame: cmi_df["county_id_fips"] = cmi_df["state_id_fips"] + cmi_df["county_id_fips"] cmi_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("coalmine_eia923") + .get_resource("core_eia923__entity_coalmine") .encode(cmi_df) ) return cmi_df @@ -560,7 +560,7 @@ def plants_eia923(eia923_dfs, eia923_transformed_dfs): def gen_fuel_nuclear(gen_fuel_nuke: pd.DataFrame) -> pd.DataFrame: - """Transforms the generation_fuel_nuclear_eia923 table. + """Transforms the core_eia923__monthly_generation_fuel_nuclear table. Transformations include: @@ -598,7 +598,7 @@ def gen_fuel_nuclear(gen_fuel_nuke: pd.DataFrame) -> pd.DataFrame: }, ) def _core_eia_923__generation_fuel_eia923(raw_eia923__generation_fuel: pd.DataFrame): - """Transforms the generation_fuel_eia923 table. + """Transforms the raw_eia923__generation_fuel table. Transformations include: @@ -617,13 +617,13 @@ def _core_eia_923__generation_fuel_eia923(raw_eia923__generation_fuel: pd.DataFr raw_eia923__generation_fuel: The raw ``raw_eia923__generation_fuel`` dataframe. Returns: - _core_eia923__generation_fuel: Cleaned ``generation_fuel_eia923`` dataframe ready for harvesting. - _core_eia923__generation_fuel_nuclear: Cleaned ``generation_fuel_nuclear_eia923`` dataframe ready for harvesting. + _core_eia923__generation_fuel: Cleaned ``eia923__generation_fuel`` dataframe ready for harvesting. + _core_eia923__generation_fuel_nuclear: Cleaned ``eia923__generation_fuel_nuclear`` dataframe ready for harvesting. """ # This needs to be a copy of what we're passed in so we can edit it. gen_fuel = raw_eia923__generation_fuel - # Drop fields we're not inserting into the generation_fuel_eia923 table. + # Drop fields we're not inserting into the _core_eia923__generation_fuel table. cols_to_drop = [ "combined_heat_power", "plant_name_eia", @@ -665,13 +665,13 @@ def _core_eia_923__generation_fuel_eia923(raw_eia923__generation_fuel: pd.DataFr gen_fuel = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("generation_fuel_eia923") + .get_resource("core_eia923__monthly_generation_fuel") .encode(gen_fuel) ) gen_fuel["fuel_type_code_pudl"] = gen_fuel.energy_source_code.map( pudl.helpers.label_map( - CODE_METADATA["energy_sources_eia"]["df"], + CODE_METADATA["core_eia__codes_energy_sources"]["df"], from_col="code", to_col="fuel_type_code_pudl", null_value=pd.NA, @@ -748,7 +748,7 @@ def _map_prime_mover_sets(prime_mover_set: np.ndarray) -> str: def _aggregate_duplicate_boiler_fuel_keys(boiler_fuel_df: pd.DataFrame) -> pd.DataFrame: """Combine boiler_fuel rows with duplicate keys by aggregating them. - Boiler_fuel_eia923 contains a few records with duplicate keys, mostly caused by + core_eia923__monthly_boiler_fuel contains a few records with duplicate keys, mostly caused by CA and CT parts of combined cycle plants being mapped to the same boiler ID. This is most likely a data entry error. See GitHub issue #852 @@ -817,7 +817,7 @@ def _aggregate_duplicate_boiler_fuel_keys(boiler_fuel_df: pd.DataFrame) -> pd.Da @asset def _core_eia923__boiler_fuel(raw_eia923__boiler_fuel: pd.DataFrame) -> pd.DataFrame: - """Transforms the boiler_fuel_eia923 table. + """Transforms the core_eia923__monthly_boiler_fuel table. Transformations include: @@ -832,7 +832,7 @@ def _core_eia923__boiler_fuel(raw_eia923__boiler_fuel: pd.DataFrame) -> pd.DataF raw_eia923__boiler_fuel: The raw ``raw_eia923__boiler_fuel`` dataframe. Returns: - Cleaned ``boiler_fuel_eia923`` dataframe ready for harvesting. + Cleaned ``core_eia923__monthly_boiler_fuel`` dataframe ready for harvesting. """ bf_df = raw_eia923__boiler_fuel @@ -868,14 +868,14 @@ def _core_eia923__boiler_fuel(raw_eia923__boiler_fuel: pd.DataFrame) -> pd.DataF bf_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("boiler_fuel_eia923") + .get_resource("core_eia923__monthly_boiler_fuel") .encode(bf_df) ) # Add a simplified PUDL fuel type bf_df["fuel_type_code_pudl"] = bf_df.energy_source_code.map( pudl.helpers.label_map( - CODE_METADATA["energy_sources_eia"]["df"], + CODE_METADATA["core_eia__codes_energy_sources"]["df"], from_col="code", to_col="fuel_type_code_pudl", null_value=pd.NA, @@ -886,7 +886,7 @@ def _core_eia923__boiler_fuel(raw_eia923__boiler_fuel: pd.DataFrame) -> pd.DataF def remove_duplicate_pks_boiler_fuel_eia923(bf: pd.DataFrame) -> pd.DataFrame: - """Deduplicate on primary keys for :ref:`boiler_fuel_eia923`. + """Deduplicate on primary keys for :ref:`core_eia923__monthly_boiler_fuel`. There are a relatively small number of records ~5% from the boiler fuel table that have duplicate records based on what we believe is this table's primary keys. @@ -904,7 +904,7 @@ def remove_duplicate_pks_boiler_fuel_eia923(bf: pd.DataFrame) -> pd.DataFrame: """ pk = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("boiler_fuel_eia923") + .get_resource("core_eia923__monthly_boiler_fuel") .schema.primary_key ) @@ -931,7 +931,7 @@ def remove_duplicate_pks_boiler_fuel_eia923(bf: pd.DataFrame) -> pd.DataFrame: ] ).empty: raise AssertionError( - f"There are ({len(pk_dupes)}) boiler_fuel_eia923 records with " + f"There are ({len(pk_dupes)}) core_eia923__monthly_boiler_fuel records with " "duplicate primary keys after cleaning - expected 0." ) return pd.concat([bf[~pk_dupe_mask], bf_no_null_pks_dupes]) @@ -939,7 +939,7 @@ def remove_duplicate_pks_boiler_fuel_eia923(bf: pd.DataFrame) -> pd.DataFrame: @asset def _core_eia923__generation(raw_eia923__generator: pd.DataFrame) -> pd.DataFrame: - """Transforms the generation_eia923 table. + """Transforms the EIA 923 generation table. Transformations include: @@ -952,7 +952,7 @@ def _core_eia923__generation(raw_eia923__generator: pd.DataFrame) -> pd.DataFram raw_eia923__generator: The raw ``raw_eia923__generator`` dataframe. Returns: - Cleaned ``generation_eia923`` dataframe ready for harvesting. + Cleaned ``_core_eia923__generation`` dataframe ready for harvesting. """ gen_df = ( raw_eia923__generator.dropna(subset=["generator_id"]) @@ -996,7 +996,7 @@ def _core_eia923__generation(raw_eia923__generator: pd.DataFrame) -> pd.DataFram gen_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("generation_eia923") + .get_resource("core_eia923__monthly_generation") .encode(gen_df) ) @@ -1007,7 +1007,7 @@ def _core_eia923__generation(raw_eia923__generator: pd.DataFrame) -> pd.DataFram def _core_eia923__coalmine( raw_eia923__fuel_receipts_costs: pd.DataFrame, ) -> pd.DataFrame: - """Transforms the coalmine_eia923 table. + """Transforms the raw_eia923__fuel_receipts_costs table. Transformations include: @@ -1016,10 +1016,10 @@ def _core_eia923__coalmine( Args: raw_eia923__fuel_receipts_costs: raw precursor to the - :ref:`fuel_receipts_costs_eia923` table. + :ref:`core_eia923__monthly_fuel_receipts_costs` table. Returns: - Cleaned ``coalmine_eia923`` dataframe ready for harvesting. + Cleaned ``_core_eia923__coalmine`` dataframe ready for harvesting. """ # These are the columns that we want to keep from FRC for the # coal mine info table. @@ -1077,7 +1077,7 @@ def _core_eia923__coalmine( cmi_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("coalmine_eia923") + .get_resource("core_eia923__entity_coalmine") .encode(cmi_df) ) @@ -1088,7 +1088,7 @@ def _core_eia923__coalmine( def _core_eia923__fuel_receipts_costs( raw_eia923__fuel_receipts_costs: pd.DataFrame, _core_eia923__coalmine: pd.DataFrame ) -> pd.DataFrame: - """Transforms the fuel_receipts_costs_eia923 dataframe. + """Transforms the eia923__fuel_receipts_costs dataframe. Transformations include: @@ -1102,14 +1102,14 @@ def _core_eia923__fuel_receipts_costs( Args: raw_eia923__fuel_receipts_costs: The raw ``raw_eia923__fuel_receipts_costs`` dataframe. - _core_eia923__coalmine: The cleaned pre-harvest ``coalmine_eia923`` dataframe. + _core_eia923__coalmine: The cleaned pre-harvest EIA 923 coal mine dataframe. Returns: - Cleaned ``fuel_receipts_costs_eia923`` dataframe ready for harvesting. + Cleaned ``eia923__fuel_receipts_costs`` dataframe ready for harvesting. """ frc_df = raw_eia923__fuel_receipts_costs - # Drop fields we're not inserting into the fuel_receipts_costs_eia923 + # Drop fields we're not inserting into the eia923__fuel_receipts_costs # table. cols_to_drop = [ "plant_name_eia", @@ -1136,7 +1136,7 @@ def _core_eia923__fuel_receipts_costs( ) # This type/naming cleanup function is separated out so that we can be - # sure it is applied exactly the same both when the coalmine_eia923 table + # sure it is applied exactly the same both when the core_eia923__entity_coalmine table # is populated, and here (since we need them to be identical for the # following merge) frc_df = ( @@ -1203,12 +1203,12 @@ def _core_eia923__fuel_receipts_costs( ) frc_df = ( pudl.metadata.classes.Package.from_resource_ids() - .get_resource("fuel_receipts_costs_eia923") + .get_resource("core_eia923__monthly_fuel_receipts_costs") .encode(frc_df) ) frc_df["fuel_type_code_pudl"] = frc_df.energy_source_code.map( pudl.helpers.label_map( - CODE_METADATA["energy_sources_eia"]["df"], + CODE_METADATA["core_eia__codes_energy_sources"]["df"], from_col="code", to_col="fuel_type_code_pudl", null_value=pd.NA, diff --git a/src/pudl/transform/epacems.py b/src/pudl/transform/epacems.py index 07171e4ee1..333dd77c9e 100644 --- a/src/pudl/transform/epacems.py +++ b/src/pudl/transform/epacems.py @@ -43,7 +43,7 @@ def harmonize_eia_epa_orispl( Args: df: A CEMS hourly dataframe for one year-month-state. - crosswalk_df: The epacamd_eia dataframe from the database. + crosswalk_df: The core_epa__assn_eia_epacamd dataframe from the database. Returns: The same data, with the ORISPL plant codes corrected to match the EIA plant IDs. @@ -57,7 +57,7 @@ def harmonize_eia_epa_orispl( ) if not one_to_many.empty: raise AssertionError( - "The epacamd_eia crosswalk has more than one plant_id_eia value per " + "The core_epa__assn_eia_epacamd crosswalk has more than one plant_id_eia value per " "plant_id_epa and emissions_unit_id_epa group" ) crosswalk_df = crosswalk_df[ @@ -106,7 +106,7 @@ def convert_to_utc(df: pd.DataFrame, plant_utc_offset: pd.DataFrame) -> pd.DataF on="plant_id_eia", ) - # Some of the timezones in the plants_entity_eia table may be missing, + # Some of the timezones in the core_eia__entity_plants table may be missing, # but none of the CEMS plants should be. if df["utc_offset"].isna().any(): missing_plants = df.loc[df["utc_offset"].isna(), "plant_id_eia"].unique() @@ -129,7 +129,7 @@ def convert_to_utc(df: pd.DataFrame, plant_utc_offset: pd.DataFrame) -> pd.DataF return df -def _load_plant_utc_offset(plants_entity_eia: pd.DataFrame) -> pd.DataFrame: +def _load_plant_utc_offset(core_eia__entity_plants: pd.DataFrame) -> pd.DataFrame: """Load the UTC offset each EIA plant. CEMS times don't change for DST, so we get the UTC offset by using the @@ -140,9 +140,9 @@ def _load_plant_utc_offset(plants_entity_eia: pd.DataFrame) -> pd.DataFrame: an existing PUDL DB. Returns: - Dataframe of applicable timezones taken from the plants_entity_eia table. + Dataframe of applicable timezones taken from the core_eia__entity_plants table. """ - timezones = plants_entity_eia[["plant_id_eia", "timezone"]].copy().dropna() + timezones = core_eia__entity_plants[["plant_id_eia", "timezone"]].copy().dropna() jan1 = datetime.datetime(2011, 1, 1) # year doesn't matter timezones["utc_offset"] = timezones["timezone"].apply( lambda tz: pytz.timezone(tz).localize(jan1).utcoffset() @@ -174,8 +174,8 @@ def correct_gross_load_mw(df: pd.DataFrame) -> pd.DataFrame: def transform( raw_df: pd.DataFrame, - epacamd_eia: pd.DataFrame, - plants_entity_eia: pd.DataFrame, + core_epa__assn_eia_epacamd: pd.DataFrame, + core_eia__entity_plants: pd.DataFrame, ) -> pd.DataFrame: """Transform EPA CEMS hourly data and ready it for export to Parquet. @@ -191,9 +191,10 @@ def transform( return ( raw_df.pipe(apply_pudl_dtypes, group="epacems") .pipe(remove_leading_zeros_from_numeric_strings, "emissions_unit_id_epa") - .pipe(harmonize_eia_epa_orispl, epacamd_eia) + .pipe(harmonize_eia_epa_orispl, core_epa__assn_eia_epacamd) .pipe( - convert_to_utc, plant_utc_offset=_load_plant_utc_offset(plants_entity_eia) + convert_to_utc, + plant_utc_offset=_load_plant_utc_offset(core_eia__entity_plants), ) .pipe(correct_gross_load_mw) .pipe(apply_pudl_dtypes, group="epacems") diff --git a/src/pudl/transform/ferc1.py b/src/pudl/transform/ferc1.py index 26f49930ca..fee273ce57 100644 --- a/src/pudl/transform/ferc1.py +++ b/src/pudl/transform/ferc1.py @@ -132,35 +132,33 @@ class TableIdFerc1(enum.Enum): Package. But this works for now. """ - FUEL_FERC1 = "fuel_ferc1" - PLANTS_STEAM_FERC1 = "plants_steam_ferc1" - PLANTS_HYDRO_FERC1 = "plants_hydro_ferc1" - PLANTS_SMALL_FERC1 = "plants_small_ferc1" - PLANTS_PUMPED_STORAGE_FERC1 = "plants_pumped_storage_ferc1" - PLANT_IN_SERVICE_FERC1 = "plant_in_service_ferc1" - PURCHASED_POWER_FERC1 = "purchased_power_ferc1" - TRANSMISSION_STATISTICS_FERC1 = "transmission_statistics_ferc1" - ELECTRIC_ENERGY_SOURCES_FERC1 = "electric_energy_sources_ferc1" - ELECTRIC_ENERGY_DISPOSITIONS_FERC1 = "electric_energy_dispositions_ferc1" - UTILITY_PLANT_SUMMARY_FERC1 = "utility_plant_summary_ferc1" - ELECTRIC_OPERATING_EXPENSES_FERC1 = "electric_operating_expenses_ferc1" - BALANCE_SHEET_LIABILITIES = "balance_sheet_liabilities_ferc1" - DEPRECIATION_AMORTIZATION_SUMMARY_FERC1 = "depreciation_amortization_summary_ferc1" - BALANCE_SHEET_ASSETS_FERC1 = "balance_sheet_assets_ferc1" - RETAINED_EARNINGS_FERC1 = "retained_earnings_ferc1" - INCOME_STATEMENT_FERC1 = "income_statement_ferc1" - ELECTRIC_PLANT_DEPRECIATION_CHANGES_FERC1 = ( - "electric_plant_depreciation_changes_ferc1" + STEAM_PLANTS_FUEL = "core_ferc1__yearly_steam_plants_fuel_sched402" + STEAM_PLANTS = "core_ferc1__yearly_steam_plants_sched402" + HYDROELECTRIC_PLANTS = "core_ferc1__yearly_hydroelectric_plants_sched406" + SMALL_PLANTS = "core_ferc1__yearly_small_plants_sched410" + PUMPED_STORAGE_PLANTS = "core_ferc1__yearly_pumped_storage_plants_sched408" + PLANT_IN_SERVICE = "core_ferc1__yearly_plant_in_service_sched204" + PURCHASED_POWER_AND_EXCHANGES = ( + "core_ferc1__yearly_purchased_power_and_exchanges_sched326" ) - ELECTRIC_OPERATING_REVENUES_FERC1 = "electric_operating_revenues_ferc1" - ELECTRIC_PLANT_DEPRECIATION_FUNCTIONAL_FERC1 = ( - "electric_plant_depreciation_functional_ferc1" + TRANSMISSION_LINES = "core_ferc1__yearly_transmission_lines_sched422" + ENERGY_SOURCES = "core_ferc1__yearly_energy_sources_sched401" + ENERGY_DISPOSITIONS = "core_ferc1__yearly_energy_dispositions_sched401" + UTILITY_PLANT_SUMMARY = "core_ferc1__yearly_utility_plant_summary_sched200" + OPERATING_EXPENSES = "core_ferc1__yearly_operating_expenses_sched320" + BALANCE_SHEET_LIABILITIES = "core_ferc1__yearly_balance_sheet_liabilities_sched110" + DEPRECIATION_SUMMARY = "core_ferc1__yearly_depreciation_summary_sched336" + BALANCE_SHEET_ASSETS = "core_ferc1__yearly_balance_sheet_assets_sched110" + RETAINED_EARNINGS = "core_ferc1__yearly_retained_earnings_sched118" + INCOME_STATEMENTS = "core_ferc1__yearly_income_statements_sched114" + DEPRECIATION_CHANGES = "core_ferc1__yearly_depreciation_changes_sched219" + OPERATING_REVENUES = "core_ferc1__yearly_operating_revenues_sched300" + DEPRECIATION_BY_FUNCTION = "core_ferc1__yearly_depreciation_by_function_sched219" + CASH_FLOWS = "core_ferc1__yearly_cash_flows_sched120" + SALES_BY_RATE_SCHEDULES = "core_ferc1__yearly_sales_by_rate_schedules_sched304" + OTHER_REGULATORY_LIABILITIES = ( + "core_ferc1__yearly_other_regulatory_liabilities_sched278" ) - CASH_FLOW_FERC1 = "cash_flow_ferc1" - ELECTRICITY_SALES_BY_RATE_SCHEDULE_FERC1 = ( - "electricity_sales_by_rate_schedule_ferc1" - ) - OTHER_REGULATORY_LIABILITIES_FERC1 = "other_regulatory_liabilities_ferc1" ################################################################################ @@ -286,7 +284,7 @@ def wide_to_tidy(df: pd.DataFrame, params: WideToTidy) -> pd.DataFrame: required for that aggregation are added later. For table that have a internal relationship between the values in the - ``params.value_types``, such as the :ref:`plant_in_service_ferc1` table, this also + ``params.value_types``, such as the :ref:`core_ferc1__yearly_plant_in_service_sched204` table, this also enables aggregation across columns to calculate the ending balance based on the starting balance and all of the reported changes. """ @@ -2862,11 +2860,11 @@ def reconcile_table_calculations( return df -class FuelFerc1TableTransformer(Ferc1AbstractTableTransformer): - """A table transformer specific to the :ref:`fuel_ferc1` table. +class SteamPlantsFuelTableTransformer(Ferc1AbstractTableTransformer): + """A table transformer specific to the :ref:`core_ferc1__yearly_steam_plants_fuel_sched402` table. - The :ref:`fuel_ferc1` table reports data about fuel consumed by large thermal power - plants in the :ref:`plants_steam_ferc1` table. Each record in the steam table is + The :ref:`core_ferc1__yearly_steam_plants_fuel_sched402` table reports data about fuel consumed by large thermal power + plants in the :ref:`core_ferc1__yearly_steam_plants_sched402` table. Each record in the steam table is typically associated with several records in the fuel table, with each fuel record reporting data for a particular type of fuel consumed by that plant over the course of a year. The fuel table presents several challenges. @@ -2918,11 +2916,11 @@ class FuelFerc1TableTransformer(Ferc1AbstractTableTransformer): * gas: reported in a mix of MMBTU/cubic foot, and MMBTU/thousand cubic feet. """ - table_id: TableIdFerc1 = TableIdFerc1.FUEL_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.STEAM_PLANTS_FUEL @cache_df(key="main") def transform_main(self, df: pd.DataFrame) -> pd.DataFrame: - """Table specific transforms for fuel_ferc1. + """Table specific transforms for core_ferc1__yearly_steam_plants_fuel_sched402. Args: df: Pre-processed, concatenated XBRL and DBF data. @@ -2960,7 +2958,7 @@ def process_dbf(self, raw_dbf: pd.DataFrame) -> pd.DataFrame: def process_xbrl( self, raw_xbrl_instant: pd.DataFrame, raw_xbrl_duration: pd.DataFrame ) -> pd.DataFrame: - """Special pre-concat treatment of the :ref:`fuel_ferc1` table. + """Special pre-concat treatment of the :ref:`core_ferc1__yearly_steam_plants_fuel_sched402` table. We have to do most of the transformation before the DBF and XBRL data have been concatenated because the fuel type column is part of the primary key and it is @@ -3170,7 +3168,7 @@ def drop_total_rows(self, df: pd.DataFrame) -> pd.DataFrame: * have no identifiable fuel units * DO report a value for MMBTU / MWh (heat rate) - In the case of the fuel_ferc1 table, we drop any row where all the data columns + In the case of the core_ferc1__yearly_steam_plants_fuel_sched402 table, we drop any row where all the data columns are null AND there's a non-null value in the ``fuel_mmbtu_per_mwh`` column, as it typically indicates a "total" row for a plant. We also require a null value for the fuel_units and an "other" value for the fuel type. @@ -3203,34 +3201,34 @@ def drop_invalid_rows( This method both drops rows in which all required data columns are null (using the inherited parameterized method) and then also drops those rows we believe - represent plant totals. See :meth:`FuelFerc1TableTransformer.drop_total_rows`. + represent plant totals. See :meth:`SteamPlantsFuelTableTransformer.drop_total_rows`. """ return super().drop_invalid_rows(df, params).pipe(self.drop_total_rows) -class PlantsSteamFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for the :ref:`plants_steam_ferc1` table.""" +class SteamPlantsTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for the :ref:`core_ferc1__yearly_steam_plants_sched402` table.""" - table_id: TableIdFerc1 = TableIdFerc1.PLANTS_STEAM_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.STEAM_PLANTS @cache_df(key="main") def transform_main( self, df: pd.DataFrame, transformed_fuel: pd.DataFrame ) -> pd.DataFrame: - """Perform table transformations for the :ref:`plants_steam_ferc1` table. + """Perform table transformations for the :ref:`core_ferc1__yearly_steam_plants_sched402` table. Note that this method has a non-standard call signature, since the - :ref:`plants_steam_ferc1` table depends on the :ref:`fuel_ferc1` table. + :ref:`core_ferc1__yearly_steam_plants_sched402` table depends on the :ref:`core_ferc1__yearly_steam_plants_fuel_sched402` table. Args: df: The pre-processed steam plants table. - transformed_fuel: The fully transformed :ref:`fuel_ferc1` table. This is + transformed_fuel: The fully transformed :ref:`core_ferc1__yearly_steam_plants_fuel_sched402` table. This is required because fuel consumption information is used to help link steam plant records together across years using :func:`plants_steam_assign_plant_ids` """ fuel_categories = list( - FuelFerc1TableTransformer() + SteamPlantsFuelTableTransformer() .params.categorize_strings["fuel_type_code_pudl"] .categories.keys() ) @@ -3278,10 +3276,10 @@ def transform( return df -class PlantsHydroFerc1TableTransformer(Ferc1AbstractTableTransformer): - """A table transformer specific to the :ref:`plants_hydro_ferc1` table.""" +class HydroelectricPlantsTableTransformer(Ferc1AbstractTableTransformer): + """A table transformer specific to the :ref:`core_ferc1__yearly_hydroelectric_plants_sched406` table.""" - table_id: TableIdFerc1 = TableIdFerc1.PLANTS_HYDRO_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.HYDROELECTRIC_PLANTS def transform_main(self, df): """Add bespoke removal of duplicate record after standard transform_main.""" @@ -3325,14 +3323,14 @@ def targeted_drop_duplicates(self, df): return df -class PlantsPumpedStorageFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`plants_pumped_storage_ferc1` table.""" +class PumpedStoragePlantsTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_pumped_storage_plants_sched408` table.""" - table_id: TableIdFerc1 = TableIdFerc1.PLANTS_PUMPED_STORAGE_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.PUMPED_STORAGE_PLANTS -class PurchasedPowerFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`purchased_power_ferc1` table. +class PurchasedPowerAndExchangesTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_purchased_power_and_exchanges_sched326`. This table has data about inter-utility power purchases into the PUDL DB. This includes how much electricty was purchased, how much it cost, and who it was @@ -3342,13 +3340,13 @@ class PurchasedPowerFerc1TableTransformer(Ferc1AbstractTableTransformer): eventually. """ - table_id: TableIdFerc1 = TableIdFerc1.PURCHASED_POWER_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.PURCHASED_POWER_AND_EXCHANGES -class PlantInServiceFerc1TableTransformer(Ferc1AbstractTableTransformer): - """A transformer for the :ref:`plant_in_service_ferc1` table.""" +class PlantInServiceTableTransformer(Ferc1AbstractTableTransformer): + """A transformer for the :ref:`core_ferc1__yearly_plant_in_service_sched204` table.""" - table_id: TableIdFerc1 = TableIdFerc1.PLANT_IN_SERVICE_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.PLANT_IN_SERVICE has_unique_record_ids: bool = False @cache_df("process_xbrl_metadata") @@ -3535,14 +3533,14 @@ def transform_main(self, df: pd.DataFrame) -> pd.DataFrame: return df -class PlantsSmallFerc1TableTransformer(Ferc1AbstractTableTransformer): - """A table transformer specific to the :ref:`plants_small_ferc1` table.""" +class SmallPlantsTableTransformer(Ferc1AbstractTableTransformer): + """A table transformer specific to the :ref:`core_ferc1__yearly_small_plants_sched410` table.""" - table_id: TableIdFerc1 = TableIdFerc1.PLANTS_SMALL_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.SMALL_PLANTS @cache_df(key="main") def transform_main(self, df: pd.DataFrame) -> pd.DataFrame: - """Table specific transforms for plants_small_ferc1. + """Table specific transforms for core_ferc1__yearly_small_plants_sched410. Params: df: Pre-processed, concatenated XBRL and DBF data. @@ -4454,10 +4452,10 @@ def spot_fix_rows(self, df: pd.DataFrame) -> pd.DataFrame: return df -class TransmissionStatisticsFerc1TableTransformer(Ferc1AbstractTableTransformer): - """A table transformer for the :ref:`transmission_statistics_ferc1` table.""" +class TransmissionLinesTableTransformer(Ferc1AbstractTableTransformer): + """A table transformer for the :ref:`core_ferc1__yearly_transmission_lines_sched422` table.""" - table_id: TableIdFerc1 = TableIdFerc1.TRANSMISSION_STATISTICS_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.TRANSMISSION_LINES has_unique_record_ids: bool = False def transform_main(self: Self, df: pd.DataFrame) -> pd.DataFrame: @@ -4466,8 +4464,8 @@ def transform_main(self: Self, df: pd.DataFrame) -> pd.DataFrame: return super().transform_main(df) -class ElectricEnergySourcesFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`electric_energy_sources_ferc1` table. +class EnergySourcesTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_energy_sources_sched401` table. The raw DBF and XBRL table will be split up into two tables. This transformer generates the sources of electricity for utilities, dropping the information about @@ -4476,7 +4474,7 @@ class ElectricEnergySourcesFerc1TableTransformer(Ferc1AbstractTableTransformer): anything with the sign. """ - table_id: TableIdFerc1 = TableIdFerc1.ELECTRIC_ENERGY_SOURCES_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.ENERGY_SOURCES has_unique_record_ids: bool = False def convert_xbrl_metadata_json_to_df( @@ -4505,17 +4503,17 @@ def convert_xbrl_metadata_json_to_df( return pd.concat([tbl_meta, new_facts]) -class ElectricEnergyDispositionsFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`electric_energy_dispositions_ferc1` table.""" +class EnergyDispositionsTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_energy_dispositions_sched401` table.""" - table_id: TableIdFerc1 = TableIdFerc1.ELECTRIC_ENERGY_DISPOSITIONS_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.ENERGY_DISPOSITIONS has_unique_record_ids: bool = False -class UtilityPlantSummaryFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`utility_plant_summary_ferc1` table.""" +class UtilityPlantSummaryTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_utility_plant_summary_sched200` table.""" - table_id: TableIdFerc1 = TableIdFerc1.UTILITY_PLANT_SUMMARY_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.UTILITY_PLANT_SUMMARY has_unique_record_ids: bool = False def process_xbrl( @@ -4579,14 +4577,15 @@ def transform_main(self: Self, df: pd.DataFrame) -> pd.DataFrame: return df def aggregated_xbrl_factoids(self: Self, df: pd.DataFrame) -> pd.DataFrame: - """Aggregate xbrl_factoids records for linking to :ref:`plant_in_service_ferc1`. + """Aggregate xbrl_factoids records for linking to :ref:`core_ferc1__yearly_plant_in_service_sched204`. This table has two ``xbrl_factoid`` which can be linked via calcuations to one - ``xbrl_factoid`` in the :ref:`plant_in_service_ferc1`. Doing this 2:1 linkage - would be fine in theory. But the :ref:`plant_in_service_ferc1` is in most senses + ``xbrl_factoid`` in the :ref:`core_ferc1__yearly_plant_in_service_sched204`. + Doing this 2:1 linkage would be fine in theory. But the + :ref:`core_ferc1__yearly_plant_in_service_sched204` is in most senses the table with the more details and of our desire to build tree-link relationships between factoids, we need to build a new factoid to link in a 1:1 - manner between this table and the :ref:`plant_in_service_ferc1`. + manner between this table and the :ref:`core_ferc1__yearly_plant_in_service_sched204`. We'll also add this factoid into the metadata via :meth:`process_xbrl_metadata` and add the linking calculation via :meth:`apply_xbrl_calculation_fixes`. @@ -4766,8 +4765,8 @@ def spot_fix_bad_signs(self: Self, df: pd.DataFrame) -> pd.DataFrame: return apply_pudl_dtypes(df, group="ferc1") -class BalanceSheetLiabilitiesFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`balance_sheet_liabilities_ferc1` table.""" +class BalanceSheetLiabilitiesTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_balance_sheet_liabilities_sched110` table.""" table_id: TableIdFerc1 = TableIdFerc1.BALANCE_SHEET_LIABILITIES has_unique_record_ids: bool = False @@ -4860,10 +4859,10 @@ def convert_xbrl_metadata_json_to_df( return pd.concat([tbl_meta, new_facts, duplicated_facts]).reset_index(drop=True) -class BalanceSheetAssetsFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`balance_sheet_assets_ferc1` table.""" +class BalanceSheetAssetsTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_balance_sheet_assets_sched110` table.""" - table_id: TableIdFerc1 = TableIdFerc1.BALANCE_SHEET_ASSETS_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.BALANCE_SHEET_ASSETS has_unique_record_ids: bool = False @cache_df("process_xbrl_metadata") @@ -4960,10 +4959,10 @@ def convert_xbrl_metadata_json_to_df( ) -class IncomeStatementFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for the :ref:`income_statement_ferc1` table.""" +class IncomeStatementsTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for the :ref:`core_ferc1__yearly_income_statements_sched114` table.""" - table_id: TableIdFerc1 = TableIdFerc1.INCOME_STATEMENT_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.INCOME_STATEMENTS has_unique_record_ids: bool = False def convert_xbrl_metadata_json_to_df( @@ -5040,10 +5039,10 @@ def transform_main(self: Self, df: pd.DataFrame) -> pd.DataFrame: return apply_pudl_dtypes(df, group="ferc1") -class RetainedEarningsFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`retained_earnings_ferc1` table.""" +class RetainedEarningsTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_retained_earnings_sched118` table.""" - table_id: TableIdFerc1 = TableIdFerc1.RETAINED_EARNINGS_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.RETAINED_EARNINGS has_unique_record_ids: bool = False current_year_types: set[str] = { @@ -5409,7 +5408,7 @@ def deduplicate_xbrl_factoid_xbrl_metadata(self, tbl_meta) -> pd.DataFrame: column. Note: This is **almost** the same as the method for - :ref:`electric_operating_revenues_ferc1`. If we wanted to lean into this + :ref:`core_ferc1__yearly_operating_revenues_sched300`. If we wanted to lean into this version of deduplication more generally this might be a fine way start to an abstraction, but ideally we wouldn't need to dedupe this at all and instead enable metadata for every value column from :meth:`wide_to_tidy`. @@ -5435,12 +5434,10 @@ def deduplicate_xbrl_factoid_xbrl_metadata(self, tbl_meta) -> pd.DataFrame: return tbl_meta_cleaned -class DepreciationAmortizationSummaryFerc1TableTransformer( - Ferc1AbstractTableTransformer -): - """Transformer class for :ref:`depreciation_amortization_summary_ferc1` table.""" +class DepreciationSummaryTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_depreciation_summary_sched336` table.""" - table_id: TableIdFerc1 = TableIdFerc1.DEPRECIATION_AMORTIZATION_SUMMARY_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.DEPRECIATION_SUMMARY has_unique_record_ids: bool = False @cache_df("process_xbrl_metadata") @@ -5485,12 +5482,10 @@ def transform_main(self, df): return df -class ElectricPlantDepreciationChangesFerc1TableTransformer( - Ferc1AbstractTableTransformer -): - """Transformer class for :ref:`electric_plant_depreciation_changes_ferc1` table.""" +class DepreciationChangesTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_depreciation_changes_sched219` table.""" - table_id: TableIdFerc1 = TableIdFerc1.ELECTRIC_PLANT_DEPRECIATION_CHANGES_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.DEPRECIATION_CHANGES has_unique_record_ids: bool = False def convert_xbrl_metadata_json_to_df( @@ -5554,12 +5549,10 @@ def process_instant_xbrl(self, df: pd.DataFrame) -> pd.DataFrame: return df -class ElectricPlantDepreciationFunctionalFerc1TableTransformer( - Ferc1AbstractTableTransformer -): - """Transformer for :ref:`electric_plant_depreciation_functional_ferc1` table.""" +class DepreciationByFunctionTableTransformer(Ferc1AbstractTableTransformer): + """Transformer for :ref:`core_ferc1__yearly_depreciation_by_function_sched219` table.""" - table_id: TableIdFerc1 = TableIdFerc1.ELECTRIC_PLANT_DEPRECIATION_FUNCTIONAL_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.DEPRECIATION_BY_FUNCTION has_unique_record_ids: bool = False def convert_xbrl_metadata_json_to_df( @@ -5659,10 +5652,10 @@ def transform_main(self, df: pd.DataFrame) -> pd.DataFrame: return df -class ElectricOperatingExpensesFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`electric_operating_expenses_ferc1` table.""" +class OperatingExpensesTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_operating_expenses_sched320` table.""" - table_id: TableIdFerc1 = TableIdFerc1.ELECTRIC_OPERATING_EXPENSES_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.OPERATING_EXPENSES has_unique_record_ids: bool = False def targeted_drop_duplicates_dbf(self, raw_df: pd.DataFrame) -> pd.DataFrame: @@ -5734,10 +5727,10 @@ def transform_main(self, df): return super().transform_main(df).assign(utility_type="electric") -class ElectricOperatingRevenuesFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`electric_operating_revenues_ferc1` table.""" +class OperatingRevenuesTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_operating_revenues_sched300` table.""" - table_id: TableIdFerc1 = TableIdFerc1.ELECTRIC_OPERATING_REVENUES_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.OPERATING_REVENUES has_unique_record_ids: bool = False def deduplicate_xbrl_factoid_xbrl_metadata( @@ -5827,10 +5820,10 @@ def targeted_drop_duplicates(self, df): return df[~dupe_mask].copy() -class CashFlowFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transform class for :ref:`cash_flow_ferc1` table.""" +class CashFlowsTableTransformer(Ferc1AbstractTableTransformer): + """Transform class for :ref:`core_ferc1__yearly_cash_flows_sched120` table.""" - table_id: TableIdFerc1 = TableIdFerc1.CASH_FLOW_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.CASH_FLOWS has_unique_record_ids: bool = False @cache_df("process_instant_xbrl") @@ -5933,12 +5926,10 @@ def convert_xbrl_metadata_json_to_df( return pd.concat([meta, ending_balance]) -class ElectricitySalesByRateScheduleFerc1TableTransformer( - Ferc1AbstractTableTransformer -): - """Transform class for :ref:`electricity_sales_by_rate_schedule_ferc1` table.""" +class SalesByRateSchedulesTableTransformer(Ferc1AbstractTableTransformer): + """Transform class for :ref:`core_ferc1__yearly_sales_by_rate_schedules_sched304` table.""" - table_id: TableIdFerc1 = TableIdFerc1.ELECTRICITY_SALES_BY_RATE_SCHEDULE_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.SALES_BY_RATE_SCHEDULES has_unique_record_ids: bool = False def add_axis_to_total_table_rows(self, df: pd.DataFrame): @@ -5991,37 +5982,37 @@ def process_xbrl( ) -class OtherRegulatoryLiabilitiesFerc1TableTransformer(Ferc1AbstractTableTransformer): - """Transformer class for :ref:`other_regulatory_liabilities_ferc1` table.""" +class OtherRegulatoryLiabilitiesTableTransformer(Ferc1AbstractTableTransformer): + """Transformer class for :ref:`core_ferc1__yearly_other_regulatory_liabilities_sched278` table.""" - table_id: TableIdFerc1 = TableIdFerc1.OTHER_REGULATORY_LIABILITIES_FERC1 + table_id: TableIdFerc1 = TableIdFerc1.OTHER_REGULATORY_LIABILITIES has_unique_record_ids = False FERC1_TFR_CLASSES: Mapping[str, type[Ferc1AbstractTableTransformer]] = { - "fuel_ferc1": FuelFerc1TableTransformer, - "plants_steam_ferc1": PlantsSteamFerc1TableTransformer, - "plants_small_ferc1": PlantsSmallFerc1TableTransformer, - "plants_hydro_ferc1": PlantsHydroFerc1TableTransformer, - "plant_in_service_ferc1": PlantInServiceFerc1TableTransformer, - "plants_pumped_storage_ferc1": PlantsPumpedStorageFerc1TableTransformer, - "transmission_statistics_ferc1": TransmissionStatisticsFerc1TableTransformer, - "purchased_power_ferc1": PurchasedPowerFerc1TableTransformer, - "electric_energy_sources_ferc1": ElectricEnergySourcesFerc1TableTransformer, - "electric_energy_dispositions_ferc1": ElectricEnergyDispositionsFerc1TableTransformer, - "utility_plant_summary_ferc1": UtilityPlantSummaryFerc1TableTransformer, - "electric_operating_expenses_ferc1": ElectricOperatingExpensesFerc1TableTransformer, - "balance_sheet_liabilities_ferc1": BalanceSheetLiabilitiesFerc1TableTransformer, - "depreciation_amortization_summary_ferc1": DepreciationAmortizationSummaryFerc1TableTransformer, - "balance_sheet_assets_ferc1": BalanceSheetAssetsFerc1TableTransformer, - "income_statement_ferc1": IncomeStatementFerc1TableTransformer, - "electric_plant_depreciation_changes_ferc1": ElectricPlantDepreciationChangesFerc1TableTransformer, - "electric_plant_depreciation_functional_ferc1": ElectricPlantDepreciationFunctionalFerc1TableTransformer, - "retained_earnings_ferc1": RetainedEarningsFerc1TableTransformer, - "electric_operating_revenues_ferc1": ElectricOperatingRevenuesFerc1TableTransformer, - "cash_flow_ferc1": CashFlowFerc1TableTransformer, - "electricity_sales_by_rate_schedule_ferc1": ElectricitySalesByRateScheduleFerc1TableTransformer, - "other_regulatory_liabilities_ferc1": OtherRegulatoryLiabilitiesFerc1TableTransformer, + "core_ferc1__yearly_steam_plants_fuel_sched402": SteamPlantsFuelTableTransformer, + "core_ferc1__yearly_steam_plants_sched402": SteamPlantsTableTransformer, + "core_ferc1__yearly_small_plants_sched410": SmallPlantsTableTransformer, + "core_ferc1__yearly_hydroelectric_plants_sched406": HydroelectricPlantsTableTransformer, + "core_ferc1__yearly_plant_in_service_sched204": PlantInServiceTableTransformer, + "core_ferc1__yearly_pumped_storage_plants_sched408": PumpedStoragePlantsTableTransformer, + "core_ferc1__yearly_transmission_lines_sched422": TransmissionLinesTableTransformer, + "core_ferc1__yearly_purchased_power_and_exchanges_sched326": PurchasedPowerAndExchangesTableTransformer, + "core_ferc1__yearly_energy_sources_sched401": EnergySourcesTableTransformer, + "core_ferc1__yearly_energy_dispositions_sched401": EnergyDispositionsTableTransformer, + "core_ferc1__yearly_utility_plant_summary_sched200": UtilityPlantSummaryTableTransformer, + "core_ferc1__yearly_operating_expenses_sched320": OperatingExpensesTableTransformer, + "core_ferc1__yearly_balance_sheet_liabilities_sched110": BalanceSheetLiabilitiesTableTransformer, + "core_ferc1__yearly_depreciation_summary_sched336": DepreciationSummaryTableTransformer, + "core_ferc1__yearly_balance_sheet_assets_sched110": BalanceSheetAssetsTableTransformer, + "core_ferc1__yearly_income_statements_sched114": IncomeStatementsTableTransformer, + "core_ferc1__yearly_depreciation_changes_sched219": DepreciationChangesTableTransformer, + "core_ferc1__yearly_depreciation_by_function_sched219": DepreciationByFunctionTableTransformer, + "core_ferc1__yearly_retained_earnings_sched118": RetainedEarningsTableTransformer, + "core_ferc1__yearly_operating_revenues_sched300": OperatingRevenuesTableTransformer, + "core_ferc1__yearly_cash_flows_sched120": CashFlowsTableTransformer, + "core_ferc1__yearly_sales_by_rate_schedules_sched304": SalesByRateSchedulesTableTransformer, + "core_ferc1__yearly_other_regulatory_liabilities_sched278": OtherRegulatoryLiabilitiesTableTransformer, } @@ -6037,7 +6028,7 @@ def ferc1_transform_asset_factory( This is a convenient way to create assets for tables that only depend on raw dbf, raw xbrl instant and duration tables and xbrl metadata. For tables with additional upstream dependencies, create a stand alone asset using an asset decorator. See - the plants_steam_ferc1 asset. + the core_ferc1__yearly_steam_plants_sched402 asset. Args: table_name: The name of the table to create an asset for. @@ -6125,7 +6116,7 @@ def create_ferc1_transform_assets() -> list[AssetsDefinition]: for table_name, tfr_class in FERC1_TFR_CLASSES.items(): # Bespoke exception. fuel must come before steam b/c fuel proportions are used to # aid in FERC plant ID assignment. - if table_name != "plants_steam_ferc1": + if table_name != "core_ferc1__yearly_steam_plants_sched402": assets.append(ferc1_transform_asset_factory(table_name, tfr_class)) return assets @@ -6134,32 +6125,34 @@ def create_ferc1_transform_assets() -> list[AssetsDefinition]: @asset(io_manager_key="pudl_sqlite_io_manager") -def plants_steam_ferc1( +def core_ferc1__yearly_steam_plants_sched402( clean_xbrl_metadata_json: dict[str, dict[str, list[dict[str, Any]]]], raw_ferc1_dbf__f1_steam: pd.DataFrame, raw_ferc1_xbrl__steam_electric_generating_plant_statistics_large_plants_402_duration: pd.DataFrame, raw_ferc1_xbrl__steam_electric_generating_plant_statistics_large_plants_402_instant: pd.DataFrame, - fuel_ferc1: pd.DataFrame, + core_ferc1__yearly_steam_plants_fuel_sched402: pd.DataFrame, ) -> pd.DataFrame: - """Create the clean plants_steam_ferc1 table. + """Create the clean core_ferc1__yearly_steam_plants_sched402 table. Args: clean_xbrl_metadata_json: XBRL metadata json for all tables. raw_ferc1_dbf__f1_steam: Raw f1_steam table. raw_ferc1_xbrl__steam_electric_generating_plant_statistics_large_plants_402_duration: raw XBRL duration table. raw_ferc1_xbrl__steam_electric_generating_plant_statistics_large_plants_402_instant: raw XBRL instant table. - fuel_ferc1: Transformed fuel_ferc1 table. + core_ferc1__yearly_steam_plants_fuel_sched402: Transformed core_ferc1__yearly_steam_plants_fuel_sched402 table. Returns: - Clean plants_steam_ferc1 table. + Clean core_ferc1__yearly_steam_plants_sched402 table. """ - df = PlantsSteamFerc1TableTransformer( - xbrl_metadata_json=clean_xbrl_metadata_json["plants_steam_ferc1"] + df = SteamPlantsTableTransformer( + xbrl_metadata_json=clean_xbrl_metadata_json[ + "core_ferc1__yearly_steam_plants_sched402" + ] ).transform( raw_dbf=raw_ferc1_dbf__f1_steam, raw_xbrl_instant=raw_ferc1_xbrl__steam_electric_generating_plant_statistics_large_plants_402_instant, raw_xbrl_duration=raw_ferc1_xbrl__steam_electric_generating_plant_statistics_large_plants_402_duration, - transformed_fuel=fuel_ferc1, + transformed_fuel=core_ferc1__yearly_steam_plants_fuel_sched402, ) return convert_cols_dtypes(df, data_source="ferc1") @@ -6210,7 +6203,7 @@ def table_dimensions_ferc1(**kwargs) -> pd.DataFrame: Compile a dataframe indicating what distinct values are observed in the data for each dimension column in association with each unique combination of ``table_name`` and ``xbrl_factoid``. E.g. for all factoids found in the - :ref:`electric_plant_depreciation_functional_ferc1` table, + :ref:`core_ferc1__yearly_depreciation_by_function_sched219` table, the only value observed for ``utility_type`` is ``electric`` and the values observed for ``plant_status`` include: ``future``, ``in_service``, ``leased`` and ``total``. @@ -6388,16 +6381,19 @@ def check_calcs_vs_table( check_for_calc_components_duplicates( calc_components, - table_names_known_dupes=["electricity_sales_by_rate_schedule_ferc1"], + table_names_known_dupes=["core_ferc1__yearly_sales_by_rate_schedules_sched304"], idx=calc_and_parent_cols, ) # check for parent/child duplicates. again need to remove the - # electricity_sales_by_rate_schedule_ferc1 table. Null hack bc comparing pandas + # core_ferc1__yearly_sales_by_rate_schedules_sched304 table. Null hack bc comparing pandas # nulls self_refs_mask = calc_components[calc_and_parent_cols].fillna("NULL HACK").apply( lambda x: all(x[col] == x[f"{col}_parent"] for col in calc_cols), axis=1 - ) & (calc_components.table_name != "electricity_sales_by_rate_schedule_ferc1") + ) & ( + calc_components.table_name + != "core_ferc1__yearly_sales_by_rate_schedules_sched304" + ) if not (parent_child_dupes := calc_components.loc[self_refs_mask]).empty: raise AssertionError( f"Found {len(parent_child_dupes)} calcuations where the parent and child " @@ -6467,7 +6463,7 @@ def check_for_calc_components_duplicates( ) -> None: """Check for duplicates calculation records. - We need to remove the electricity_sales_by_rate_schedule_ferc1 bc there are + We need to remove the core_ferc1__yearly_sales_by_rate_schedules_sched304 bc there are duplicate renamed factoids in that table (originally billed/unbilled). """ calc_components_test = ( @@ -6500,13 +6496,14 @@ def make_calculation_dimensions_explicit( We have extended this calculation system to allow independent calculations to be specified for different values within a given dimension. For example, the - :ref:`utility_plant_summary_ferc1` table contains records with a variety of + :ref:`core_ferc1__yearly_utility_plant_summary_sched200` table contains records with a variety of different ``utility_type`` values (gas, electric, etc.). For many combinations of fact and ``utility_type``, no more detailed information about the soruce of the data is available, but for some, and only in the case of electric utilities, much more - detail can be found in the :ref:`plant_in_service_ferc1` table. In order to use this - additional information when it is available, we sometimes explicitly specify - different calculations for different values of additional dimension columns. + detail can be found in the :ref:`core_ferc1__yearly_plant_in_service_sched204` table. + In order to use this additional information when it is available, we sometimes + explicitly specify different calculations for different values of additional + dimension columns. This function uses the observed associations between ``table_name``, ``xbrl_factoid`` and the other dimension columns compiled by @@ -6747,7 +6744,7 @@ def infer_intra_factoid_totals( check_for_calc_components_duplicates( calcs_with_totals, table_names_known_dupes=[ - "electricity_sales_by_rate_schedule_ferc1", + "core_ferc1__yearly_sales_by_rate_schedules_sched304", ], idx=parent_node_pk + child_node_pk, ) @@ -6767,16 +6764,16 @@ def infer_intra_factoid_totals( # minus electric_plant_depreciation_changes_ferc1 bc that table is messy and # not actually in the explosion work for table_name in [ - "plant_in_service_ferc1", - "utility_plant_summary_ferc1", - "electric_operating_expenses_ferc1", - "balance_sheet_liabilities_ferc1", - "depreciation_amortization_summary_ferc1", - "balance_sheet_assets_ferc1", - "income_statement_ferc1", - "electric_plant_depreciation_functional_ferc1", - "retained_earnings_ferc1", - "electric_operating_revenues_ferc1", + "core_ferc1__yearly_plant_in_service_sched204", + "core_ferc1__yearly_utility_plant_summary_sched200", + "core_ferc1__yearly_operating_expenses_sched320", + "core_ferc1__yearly_balance_sheet_liabilities_sched110", + "core_ferc1__yearly_depreciation_summary_sched336", + "core_ferc1__yearly_balance_sheet_assets_sched110", + "core_ferc1__yearly_income_statements_sched114", + "core_ferc1__yearly_depreciation_by_function_sched219", + "core_ferc1__yearly_retained_earnings_sched118", + "core_ferc1__yearly_operating_revenues_sched300", ] } | { diff --git a/src/pudl/transform/ferc714.py b/src/pudl/transform/ferc714.py index ad07d40e56..4051738905 100644 --- a/src/pudl/transform/ferc714.py +++ b/src/pudl/transform/ferc714.py @@ -230,11 +230,11 @@ """Overrides of FERC 714 respondent IDs with wrong or missing EIA Codes.""" RENAME_COLS = { - "respondent_id_ferc714": { + "core_ferc714__respondent_id": { "respondent_id": "respondent_id_ferc714", "respondent_name": "respondent_name_ferc714", }, - "demand_hourly_pa_ferc714": { + "core_ferc714__hourly_demand_pa": { "report_yr": "report_year", "plan_date": "report_date", "respondent_id": "respondent_id_ferc714", @@ -369,7 +369,9 @@ def _standardize_offset_codes(df: pd.DataFrame, offset_fixes) -> pd.DataFrame: @asset(io_manager_key="pudl_sqlite_io_manager") -def respondent_id_ferc714(raw_ferc714__respondent_id: pd.DataFrame) -> pd.DataFrame: +def core_ferc714__respondent_id( + raw_ferc714__respondent_id: pd.DataFrame, +) -> pd.DataFrame: """Transform the FERC 714 respondent IDs, names, and EIA utility IDs. Clean up FERC-714 respondent names and manually assign EIA utility IDs to a few FERC @@ -383,17 +385,19 @@ def respondent_id_ferc714(raw_ferc714__respondent_id: pd.DataFrame) -> pd.DataFr Returns: A clean(er) version of the FERC-714 respondents table. """ - df = _pre_process(raw_ferc714__respondent_id, table_name="respondent_id_ferc714") + df = _pre_process( + raw_ferc714__respondent_id, table_name="core_ferc714__respondent_id" + ) df["respondent_name_ferc714"] = df.respondent_name_ferc714.str.strip() df.loc[df.eia_code == 0, "eia_code"] = pd.NA # There are a few utilities that seem mappable, but missing: for rid in EIA_CODE_FIXES: df.loc[df.respondent_id_ferc714 == rid, "eia_code"] = EIA_CODE_FIXES[rid] - return _post_process(df, table_name="respondent_id_ferc714") + return _post_process(df, table_name="core_ferc714__respondent_id") @asset(io_manager_key="pudl_sqlite_io_manager") -def demand_hourly_pa_ferc714( +def core_ferc714__hourly_demand_pa( raw_ferc714__demand_hourly_pa: pd.DataFrame, ) -> pd.DataFrame: """Transform the hourly demand time series by Planning Area. @@ -416,7 +420,7 @@ def demand_hourly_pa_ferc714( """ logger.info("Converting dates into pandas Datetime types.") df = _pre_process( - raw_ferc714__demand_hourly_pa, table_name="demand_hourly_pa_ferc714" + raw_ferc714__demand_hourly_pa, table_name="core_ferc714__hourly_demand_pa" ) # Parse date strings @@ -516,5 +520,5 @@ def demand_hourly_pa_ferc714( "demand_mwh", ] df = df.drop(columns=set(df.columns) - set(columns)) - df = _post_process(df[columns], table_name="demand_hourly_pa_ferc714") + df = _post_process(df[columns], table_name="core_ferc714__hourly_demand_pa") return df diff --git a/src/pudl/transform/params/__init__.py b/src/pudl/transform/params/__init__.py index 9c3c195878..3c44632dce 100644 --- a/src/pudl/transform/params/__init__.py +++ b/src/pudl/transform/params/__init__.py @@ -4,7 +4,7 @@ ``ferc1`` or ``eia923``) and must define a dictionary named ``TRANSFORM_PARAMS``. This dictionary is a nested data structure with 2 or 3 levels of keys: -* The first level has keys that table names (e.g. ``plants_steam_ferc1``). +* The first level has keys that table names (e.g. ``core_ferc1__yearly_steam_plants_sched402``). * The second level has keys that are the names of transform functions (e.g. ``convert_units``). * In the case of transform functions that operate on a single column and implement the diff --git a/src/pudl/transform/params/ferc1.py b/src/pudl/transform/params/ferc1.py index fb557ab626..edf7d3593e 100644 --- a/src/pudl/transform/params/ferc1.py +++ b/src/pudl/transform/params/ferc1.py @@ -2159,7 +2159,7 @@ # Fully assembled set of FERC 1 transformation parameters ############################################################################## TRANSFORM_PARAMS = { - "fuel_ferc1": { + "core_ferc1__yearly_steam_plants_fuel_sched402": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -2245,7 +2245,7 @@ }, ], }, - "plants_steam_ferc1": { + "core_ferc1__yearly_steam_plants_sched402": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -2414,7 +2414,7 @@ }, ], }, - "plants_hydro_ferc1": { + "core_ferc1__yearly_hydroelectric_plants_sched406": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -2552,7 +2552,7 @@ }, ], }, - "plants_small_ferc1": { + "core_ferc1__yearly_small_plants_sched410": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -2650,7 +2650,7 @@ }, ], }, - "plant_in_service_ferc1": { + "core_ferc1__yearly_plant_in_service_sched204": { "rename_columns_ferc1": { "xbrl": { "columns": { @@ -2752,7 +2752,7 @@ }, }, }, - "plants_pumped_storage_ferc1": { + "core_ferc1__yearly_pumped_storage_plants_sched408": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -2897,7 +2897,7 @@ }, ], }, - "purchased_power_ferc1": { + "core_ferc1__yearly_purchased_power_and_exchanges_sched326": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -2976,7 +2976,7 @@ } ], }, - "transmission_statistics_ferc1": { + "core_ferc1__yearly_transmission_lines_sched422": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3058,7 +3058,7 @@ } ], }, - "electric_energy_sources_ferc1": { + "core_ferc1__yearly_energy_sources_sched401": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3135,7 +3135,7 @@ }, "align_row_numbers_dbf": {"dbf_table_names": ["f1_elctrc_erg_acct"]}, }, - "electric_energy_dispositions_ferc1": { + "core_ferc1__yearly_energy_dispositions_sched401": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3200,7 +3200,7 @@ }, "align_row_numbers_dbf": {"dbf_table_names": ["f1_elctrc_erg_acct"]}, }, - "utility_plant_summary_ferc1": { + "core_ferc1__yearly_utility_plant_summary_sched200": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3374,7 +3374,7 @@ }, }, }, - "balance_sheet_assets_ferc1": { + "core_ferc1__yearly_balance_sheet_assets_sched110": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3416,7 +3416,7 @@ }, "drop_duplicate_rows_dbf": { "data_columns": ["ending_balance", "starting_balance"], - "table_name": "balance_sheet_assets_ferc1", + "table_name": "core_ferc1__yearly_balance_sheet_assets_sched110", }, "align_row_numbers_dbf": {"dbf_table_names": ["f1_comp_balance_db"]}, "merge_xbrl_metadata": { @@ -3432,7 +3432,7 @@ }, }, }, - "balance_sheet_liabilities_ferc1": { + "core_ferc1__yearly_balance_sheet_liabilities_sched110": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3475,7 +3475,7 @@ }, "drop_duplicate_rows_dbf": { "data_columns": ["ending_balance", "starting_balance"], - "table_name": "balance_sheet_liabilities_ferc1", + "table_name": "core_ferc1__yearly_balance_sheet_liabilities_sched110", }, "align_row_numbers_dbf": {"dbf_table_names": ["f1_bal_sheet_cr"]}, "merge_xbrl_metadata": { @@ -3484,7 +3484,7 @@ }, "reconcile_table_calculations": {"column_to_check": "ending_balance"}, }, - "depreciation_amortization_summary_ferc1": { + "core_ferc1__yearly_depreciation_summary_sched336": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3564,7 +3564,7 @@ }, }, }, - "electric_operating_revenues_ferc1": { + "core_ferc1__yearly_operating_revenues_sched300": { "rename_columns_ferc1": { "duration_xbrl": { "columns": { @@ -3711,7 +3711,7 @@ }, }, }, - "retained_earnings_ferc1": { + "core_ferc1__yearly_retained_earnings_sched118": { "rename_columns_ferc1": { "duration_xbrl": { "columns": { @@ -3792,7 +3792,7 @@ }, }, }, - "income_statement_ferc1": { + "core_ferc1__yearly_income_statements_sched114": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -3924,7 +3924,7 @@ }, "drop_duplicate_rows_dbf": { "data_columns": ["dollar_value"], - "table_name": "income_statement_ferc1", + "table_name": "core_ferc1__yearly_income_statements_sched114", }, "align_row_numbers_dbf": { "dbf_table_names": ["f1_income_stmnt", "f1_incm_stmnt_2"] @@ -3968,7 +3968,7 @@ }, }, }, - "electric_plant_depreciation_changes_ferc1": { + "core_ferc1__yearly_depreciation_changes_sched219": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -4089,7 +4089,7 @@ }, }, }, - "electric_plant_depreciation_functional_ferc1": { + "core_ferc1__yearly_depreciation_by_function_sched219": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -4190,7 +4190,7 @@ "subtotal_column": "plant_status", }, }, - "cash_flow_ferc1": { + "core_ferc1__yearly_cash_flows_sched120": { "rename_columns_ferc1": { "instant_xbrl": { "columns": { @@ -4311,7 +4311,7 @@ "on": "amount_type", }, }, - "electric_operating_expenses_ferc1": { + "core_ferc1__yearly_operating_expenses_sched320": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -4528,7 +4528,7 @@ }, "drop_duplicate_rows_dbf": { "data_columns": ["dollar_value"], - "table_name": "electric_operating_expenses_ferc1", + "table_name": "core_ferc1__yearly_operating_expenses_sched320", }, "merge_xbrl_metadata": { "rename_columns": {"xbrl_factoid": "expense_type"}, @@ -4552,7 +4552,7 @@ }, }, }, - "other_regulatory_liabilities_ferc1": { + "core_ferc1__yearly_other_regulatory_liabilities_sched278": { "rename_columns_ferc1": { "dbf": { "columns": { @@ -4600,7 +4600,7 @@ }, ], }, - "electricity_sales_by_rate_schedule_ferc1": { + "core_ferc1__yearly_sales_by_rate_schedules_sched304": { "rename_columns_ferc1": { "dbf": { "columns": { diff --git a/src/pudl/validate.py b/src/pudl/validate.py index 816212c8fa..d226f88420 100644 --- a/src/pudl/validate.py +++ b/src/pudl/validate.py @@ -659,7 +659,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): ############################################################################### -plants_steam_ferc1_capacity = [ +core_ferc1__yearly_steam_plants_sched402_capacity = [ { "title": "All Plant Capacity", "query": "", @@ -682,7 +682,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): }, ] -plants_steam_ferc1_expenses = [ +core_ferc1__yearly_steam_plants_sched402_expenses = [ { "title": "Capital Expenses (median)", "query": "", @@ -745,7 +745,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): }, ] -plants_steam_ferc1_capacity_ratios = [ +core_ferc1__yearly_steam_plants_sched402_capacity_ratios = [ { "title": "Capacity Factor (Tails)", "query": "capacity_factor>0.05", @@ -848,7 +848,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): }, ] -plants_steam_ferc1_connected_hours = [ +core_ferc1__yearly_steam_plants_sched402_connected_hours = [ { # Currently failing b/c ~10% of plants have way more than 8760 hours... "title": "Plant Hours Connected (min/max)", "query": "", @@ -861,7 +861,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): }, ] -plants_steam_ferc1_self = [ +core_ferc1__yearly_steam_plants_sched402_self = [ { "title": "All Plant Capacity", "query": "", @@ -2661,7 +2661,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): "low_bound": 7.0, "hi_q": 0.50, "hi_bound": 7.5, - "data_col": "heat_rate_mmbtu_mwh", + "data_col": "unit_heat_rate_mmbtu_per_mwh", "weight_col": "net_generation_mwh", }, { # EIA natural gas reporting really only becomes usable in 2015. @@ -2671,7 +2671,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): "low_bound": 6.4, "hi_q": 0.95, "hi_bound": 13.0, - "data_col": "heat_rate_mmbtu_mwh", + "data_col": "unit_heat_rate_mmbtu_per_mwh", "weight_col": "net_generation_mwh", }, ] @@ -2685,7 +2685,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): "low_bound": 10.0, "hi_q": 0.50, "hi_bound": 11.0, - "data_col": "heat_rate_mmbtu_mwh", + "data_col": "unit_heat_rate_mmbtu_per_mwh", "weight_col": "net_generation_mwh", }, { @@ -2695,7 +2695,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): "low_bound": 9.0, "hi_q": 0.95, "hi_bound": 12.5, - "data_col": "heat_rate_mmbtu_mwh", + "data_col": "unit_heat_rate_mmbtu_per_mwh", "weight_col": "net_generation_mwh", }, ] @@ -2861,7 +2861,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): "low_q": 0.05, "mid_q": 0.50, "hi_q": 0.95, - "data_col": "heat_rate_mmbtu_mwh", + "data_col": "unit_heat_rate_mmbtu_per_mwh", "weight_col": "net_generation_mwh", }, { @@ -2870,7 +2870,7 @@ def plot_vs_agg(orig_df, agg_df, validation_cases): "low_q": 0.05, "mid_q": 0.50, "hi_q": 0.95, - "data_col": "heat_rate_mmbtu_mwh", + "data_col": "unit_heat_rate_mmbtu_per_mwh", "weight_col": "net_generation_mwh", }, ] diff --git a/test/integration/datasette_metadata_test.py b/test/integration/datasette_metadata_test.py index ce0c3bf004..63676e1a03 100644 --- a/test/integration/datasette_metadata_test.py +++ b/test/integration/datasette_metadata_test.py @@ -48,7 +48,7 @@ def test_datasette_metadata_to_yml(ferc1_engine_xbrl): == "https://github.com/catalyst-cooperative/pudl" ) assert ( - parsed_metadata["databases"]["pudl"]["tables"]["plants_entity_eia"][ + parsed_metadata["databases"]["pudl"]["tables"]["core_eia__entity_plants"][ "label_column" ] == "plant_name_eia" diff --git a/test/integration/epacems_test.py b/test/integration/epacems_test.py index 3e95c5a1be..42184f2b56 100644 --- a/test/integration/epacems_test.py +++ b/test/integration/epacems_test.py @@ -29,7 +29,10 @@ def epacems_parquet_path(pudl_engine: sa.Engine) -> Path: pudl_engine: An implicit dependency that ensures the .parquet files exist. """ context = build_init_resource_context() - return epacems_io_manager(context)._base_path / "hourly_emissions_epacems.parquet" + return ( + epacems_io_manager(context)._base_path + / "core_epacems__hourly_emissions.parquet" + ) def test_epacems_subset(epacems_settings: EpaCemsSettings, epacems_parquet_path: Path): diff --git a/test/integration/etl_test.py b/test/integration/etl_test.py index 6f5c6dda72..affa1d813b 100644 --- a/test/integration/etl_test.py +++ b/test/integration/etl_test.py @@ -28,8 +28,8 @@ def test_pudl_engine( """ assert isinstance(pudl_engine, sa.Engine) insp = sa.inspect(pudl_engine) - assert "plants_pudl" in insp.get_table_names() - assert "utilities_pudl" in insp.get_table_names() + assert "core_pudl__entity_plants_pudl" in insp.get_table_names() + assert "core_pudl__entity_utilities_pudl" in insp.get_table_names() if check_foreign_keys: # Raises ForeignKeyErrors if there are any @@ -50,13 +50,15 @@ def test_ferc1_xbrl2sqlite(ferc1_engine_xbrl: sa.Engine, ferc1_xbrl_taxonomy_met # Has the metadata we've read in from JSON contain a long list of entities? assert isinstance(ferc1_xbrl_taxonomy_metadata, dict) - assert "plants_steam_ferc1" in ferc1_xbrl_taxonomy_metadata + assert "core_ferc1__yearly_steam_plants_sched402" in ferc1_xbrl_taxonomy_metadata assert len(ferc1_xbrl_taxonomy_metadata) > 10 assert len(ferc1_xbrl_taxonomy_metadata) < 100 # Can we normalize that list of entities and find data in it that we expect? df = pd.json_normalize( - ferc1_xbrl_taxonomy_metadata["plant_in_service_ferc1"]["instant"] + ferc1_xbrl_taxonomy_metadata["core_ferc1__yearly_plant_in_service_sched204"][ + "instant" + ] ) assert ( df.loc[ diff --git a/test/integration/ferc1_eia_train_test.py b/test/integration/ferc1_eia_train_test.py index ddde78d521..54f0ecf238 100644 --- a/test/integration/ferc1_eia_train_test.py +++ b/test/integration/ferc1_eia_train_test.py @@ -1,4 +1,4 @@ -"""Tests for the validation functions in the ferc1_eia_train module. +"""Tests for the validation functions in the eia_ferc1_train module. The functions that this module is testing were not designed with tests in mind. Ideally we would refactor the functions to be more testable, but for now we are just focusing on @@ -19,10 +19,10 @@ import pandas as pd import pytest -from pudl.analysis.ferc1_eia_record_linkage import ( +from pudl.analysis.eia_ferc1_record_linkage import ( restrict_train_connections_on_date_range, ) -from pudl.analysis.ferc1_eia_train import ( +from pudl.analysis.eia_ferc1_train import ( generate_all_override_spreadsheets, validate_override_fixes, ) @@ -43,16 +43,17 @@ def plant_parts_eia(fast_out_annual): @pytest.fixture(scope="module") -def ferc1_eia(fast_out_annual): - """The ferc1_eia output table.""" +def eia_ferc1(fast_out_annual): + """The eia_ferc1 output table.""" + # the pudl_out name here is the old, non-alphabetized ordering. return fast_out_annual.ferc1_eia() @pytest.fixture(scope="module") -def ferc1_eia_training_data(): - """The training data for the ferc1_eia matching.""" +def eia_ferc1_training_data(): + """The training data for the eia_ferc1 matching.""" return pd.read_csv( - importlib.resources.files("pudl.package_data.glue") / "ferc1_eia_train.csv" + importlib.resources.files("pudl.package_data.glue") / "eia_ferc1_train.csv" ) @@ -129,8 +130,8 @@ def ferc1_eia_training_data(): ) def test_validate_override_fixes( plant_parts_eia, - ferc1_eia, - ferc1_eia_training_data, + eia_ferc1, + eia_ferc1_training_data, verified, report_year, record_id_eia_override_1, @@ -147,27 +148,27 @@ def test_validate_override_fixes( "utility_id_pudl_ferc1": utility_id_pudl_ferc1, } ).assign(verified=lambda x: x.verified.astype("bool")) - ferc1_eia_training_data_restricted = restrict_train_connections_on_date_range( - train_df=ferc1_eia_training_data, + eia_ferc1_training_data_restricted = restrict_train_connections_on_date_range( + train_df=eia_ferc1_training_data, id_col="record_id_ferc1", - start_date=min(ferc1_eia.report_date), - end_date=max(ferc1_eia.report_date), + start_date=min(eia_ferc1.report_date), + end_date=max(eia_ferc1.report_date), ) validate_override_fixes( validated_connections=test_df, ppe=plant_parts_eia, - ferc1_eia=ferc1_eia, - training_data=ferc1_eia_training_data_restricted, + eia_ferc1=eia_ferc1, + training_data=eia_ferc1_training_data_restricted, expect_override_overrides=True, allow_mismatched_utilities=True, ) -def test_generate_all_override_spreadsheets(plant_parts_eia, ferc1_eia, utils_eia860): +def test_generate_all_override_spreadsheets(plant_parts_eia, eia_ferc1, utils_eia860): """Test the genation of the override spreadsheet for mapping FERC-EIA records.""" # Create the test spreadsheet generate_all_override_spreadsheets( - ferc1_eia, + eia_ferc1, plant_parts_eia, utils_eia860, util_dict={"NextEra": [6452, 7801]}, diff --git a/test/integration/glue_test.py b/test/integration/glue_test.py index 6026adaf05..5d0c08f4c9 100644 --- a/test/integration/glue_test.py +++ b/test/integration/glue_test.py @@ -75,11 +75,11 @@ def glue_test_dfs( glue_test_dfs.update( { "utilities_ferc1_dbf_labeled": label_utilities_ferc1_dbf( - glue_test_dfs["utilities_ferc1_dbf"], + glue_test_dfs["core_pudl__assn_ferc1_dbf_pudl_utilities"], glue_test_dfs["util_ids_ferc1_raw_dbf"], ), "utilities_ferc1_xbrl_labeled": label_utilities_ferc1_xbrl( - glue_test_dfs["utilities_ferc1_xbrl"], + glue_test_dfs["core_pudl__assn_ferc1_xbrl_pudl_utilities"], glue_test_dfs["util_ids_ferc1_raw_xbrl"], ), } @@ -103,63 +103,63 @@ def save_to_devtools_glue(missing_df: pd.DataFrame, test_dir, file_name: str): "ids_left,ids_right,id_cols,label_df", [ pytest.param( - "utilities_pudl", - "utilities_ferc1", + "core_pudl__entity_utilities_pudl", + "core_pudl__assn_ferc1_pudl_utilities", ["utility_id_pudl"], - "utilities_ferc1", + "core_pudl__assn_ferc1_pudl_utilities", id="missing_utility_id_pudl_in_utilities_ferc1", ), pytest.param( - "utilities_ferc1", - "utilities_ferc1_dbf", + "core_pudl__assn_ferc1_pudl_utilities", + "core_pudl__assn_ferc1_dbf_pudl_utilities", ["utility_id_ferc1"], "utilities_ferc1_dbf_labeled", id="missing_utility_id_ferc1_in_utilities_ferc1_dbf", ), pytest.param( - "utilities_ferc1", - "utilities_ferc1_xbrl", + "core_pudl__assn_ferc1_pudl_utilities", + "core_pudl__assn_ferc1_xbrl_pudl_utilities", ["utility_id_ferc1"], "utilities_ferc1_xbrl_labeled", id="missing_utility_id_ferc1_in_utilities_ferc1_xbrl", ), pytest.param( - "utilities_ferc1", - "plants_ferc1", + "core_pudl__assn_ferc1_pudl_utilities", + "core_pudl__assn_ferc1_pudl_plants", ["utility_id_ferc1"], - "plants_ferc1", + "core_pudl__assn_ferc1_pudl_plants", id="missing_utility_id_ferc1_in_plants_ferc1", ), pytest.param( - "utilities_ferc1_xbrl", + "core_pudl__assn_ferc1_xbrl_pudl_utilities", "util_ids_ferc1_raw_xbrl", ["utility_id_ferc1_xbrl"], "util_ids_ferc1_raw_xbrl", id="missing_utility_id_ferc1_xbrl_in_raw_xbrl", ), pytest.param( - "utilities_ferc1_dbf", + "core_pudl__assn_ferc1_dbf_pudl_utilities", "util_ids_ferc1_raw_dbf", ["utility_id_ferc1_dbf"], "util_ids_ferc1_raw_dbf", id="missing_utility_id_ferc1_dbf_in_raw_dbf", ), pytest.param( - "plants_pudl", - "plants_ferc1", + "core_pudl__entity_plants_pudl", + "core_pudl__assn_ferc1_pudl_plants", ["plant_id_pudl"], None, # should only ever happen if a plant is in the mapping sheet w/o a pudl id id="missing_plant_id_pudl_in_plants_ferc1", ), pytest.param( - "plants_ferc1", + "core_pudl__assn_ferc1_pudl_plants", "plants_ferc1_raw", ["utility_id_ferc1", "plant_name_ferc1"], "plants_ferc1_raw", id="missing_plants_in_plants_ferc1", ), pytest.param( - "plants_eia", + "core_pudl__assn_eia_pudl_plants", "plants_eia_pudl_db", ["plant_id_eia"], "plants_eia_labeled", @@ -222,15 +222,15 @@ def test_for_fk_validation_and_unmapped_ids( "ids_left,ids_right,id_cols,drop", [ pytest.param( - "plants_ferc1", + "core_pudl__assn_ferc1_pudl_plants", "plants_ferc1_raw", ["utility_id_ferc1", "plant_name_ferc1"], (227, "comanche"), id="check_for_unmmapped_plants_in_plants_ferc1", ), pytest.param( - "utilities_ferc1", - "utilities_ferc1_xbrl", + "core_pudl__assn_ferc1_pudl_utilities", + "core_pudl__assn_ferc1_xbrl_pudl_utilities", ["utility_id_ferc1"], (227), id="validate_utility_id_ferc1_in_utilities_ferc1_xbrl", @@ -271,12 +271,12 @@ def test_unmapped_utils_eia( This test has its own call signature because its more complex. In order to label the missing utilities, we use both the ``pudl_out`` object as well as direct SQL - queries. This test is duplicative with the sql foriegn key constraints. + queries. This test is duplicative with the sql foreign key constraints. """ unmapped_utils_eia = get_util_ids_eia_unmapped( pudl_out, pudl_engine, - glue_test_dfs["utilities_eia"], + glue_test_dfs["core_pudl__assn_eia_pudl_utilities"], ) if save_unmapped_ids: diff --git a/test/integration/output_test.py b/test/integration/output_test.py index 9439d405dd..9452cf62a3 100644 --- a/test/integration/output_test.py +++ b/test/integration/output_test.py @@ -160,8 +160,8 @@ def test_outputs_by_table_suffix(fast_out, table_suffix): @pytest.mark.parametrize( "df_name", [ - "summarized_demand_ferc714", - "fipsified_respondents_ferc714", + "out_ferc714__summarized_demand", + "out_ferc714__respondents_with_fips", ], ) def test_ferc714_outputs(pudl_engine, df_name): @@ -175,8 +175,8 @@ def test_ferc714_outputs(pudl_engine, df_name): @pytest.mark.parametrize( "df_name", [ - "compiled_geometry_balancing_authority_eia861", - "compiled_geometry_utility_eia861", + "out_eia861__compiled_geometry_balancing_authorities", + "out_eia861__compiled_geometry_utilities", ], ) def test_service_territory_outputs(pudl_engine, df_name): @@ -190,7 +190,7 @@ def test_service_territory_outputs(pudl_engine, df_name): @pytest.mark.parametrize( "df_name", [ - "predicted_state_hourly_demand", + "out_ferc714__hourly_predicted_state_demand", ], ) def test_state_demand_outputs(pudl_engine, df_name): diff --git a/test/unit/analysis/plant_parts_eia_test.py b/test/unit/analysis/plant_parts_eia_test.py index 202822c442..9b234464bf 100644 --- a/test/unit/analysis/plant_parts_eia_test.py +++ b/test/unit/analysis/plant_parts_eia_test.py @@ -534,7 +534,7 @@ def test_one_to_many(): "net_generation_mwh": [100] * 8, "total_fuel_cost": [100] * 8, "fuel_cost_per_mwh": [1] * 8, - "heat_rate_mmbtu_mwh": [1] * 8, + "unit_heat_rate_mmbtu_per_mwh": [1] * 8, "fuel_cost_per_mmbtu": [1] * 8, "fuel_type_code_pudl": ["test"] * 8, "planned_generator_retirement_date": [2076] * 8, @@ -621,7 +621,7 @@ def test_one_to_many(): "net_generation_mwh": [100, 100, 100, 100, 100, 100, 100, 100, 200], "total_fuel_cost": [100, 100, 100, 100, 100, 100, 100, 100, 200], "fuel_cost_per_mwh": [1] * 9, - "heat_rate_mmbtu_mwh": [1] * 9, + "unit_heat_rate_mmbtu_per_mwh": [1] * 9, "fuel_cost_per_mmbtu": [1] * 9, "fuel_type_code_pudl": ["test"] * 9, "planned_generator_retirement_date": [2076] * 9, diff --git a/test/unit/harvest_test.py b/test/unit/harvest_test.py index 2512f36bd7..05337cbe5d 100644 --- a/test/unit/harvest_test.py +++ b/test/unit/harvest_test.py @@ -112,7 +112,7 @@ def test_resource_with_only_key_fields_harvests() -> None: # ---- EIA example ---- # INPUT_DFS: dict[str, pd.DataFrame] = { - "service_territory_eia861": pd.DataFrame( + "core_eia861__yearly_service_territory": pd.DataFrame( columns=[ "utility_id_eia", "utility_name_eia", @@ -127,7 +127,7 @@ def test_resource_with_only_key_fields_harvests() -> None: (3989, "City of Colorado Springs - (CO)", "2018-01-01", "Teller", "CO"), ], ), - "sales_eia861": pd.DataFrame( + "core_eia861__yearly_sales": pd.DataFrame( columns=[ "utility_id_eia", "utility_name_eia", @@ -179,7 +179,7 @@ def test_resource_with_only_key_fields_harvests() -> None: ), ], ), - "generation_eia923": pd.DataFrame( + "core_eia923__monthly_generation": pd.DataFrame( columns=[ "plant_id_eia", "generator_id", @@ -204,7 +204,7 @@ def test_resource_with_only_key_fields_harvests() -> None: (3, "1", "2018-12-01", -494.0, "ST", "T", "SOCO"), ], ), - "generators_eia860": pd.DataFrame( + "core_eia860__scd_generators": pd.DataFrame( columns=[ "plant_id_eia", "generator_id", @@ -223,7 +223,7 @@ def test_resource_with_only_key_fields_harvests() -> None: (3, "2", "2018-01-01", 50, "ST", 195, "Alabama Power Co", "B", "AL"), ], ), - "boiler_generator_assn_eia860": pd.DataFrame( + "core_eia860__assn_boiler_generator": pd.DataFrame( columns=["plant_id_eia", "generator_id", "report_year", "boiler_id"], data=[ (3, "1", "2018-01-01", "1ST"), @@ -279,7 +279,7 @@ def test_resource_with_only_key_fields_harvests() -> None: }, }, { - "name": "generators_eia860", + "name": "core_eia860__scd_generators", "harvest": {"harvest": True}, "schema": { "fields": ["plant_id_eia", "generator_id", "report_year", "capacity_mw"], @@ -303,7 +303,7 @@ def test_resource_with_only_key_fields_harvests() -> None: }, }, { - "name": "generation_eia923", + "name": "core_eia923__monthly_generation", "harvest": {"harvest": False}, "schema": { "fields": [ @@ -316,7 +316,7 @@ def test_resource_with_only_key_fields_harvests() -> None: }, }, { - "name": "sales_eia861", + "name": "core_eia861__yearly_sales", "harvest": {"harvest": False}, "schema": { "fields": ["utility_id_eia", "report_year", "state", "county", "sales"], @@ -324,7 +324,7 @@ def test_resource_with_only_key_fields_harvests() -> None: }, }, { - "name": "boiler_generator_assn_eia860", + "name": "core_eia860__assn_boiler_generator", "harvest": {"harvest": True}, "schema": { "fields": ["plant_id_eia", "generator_id", "report_year", "boiler_id"], @@ -359,7 +359,7 @@ def test_resource_with_only_key_fields_harvests() -> None: (4, "b", np.nan, np.nan), ], ), - "generators_eia860": pd.DataFrame( + "core_eia860__scd_generators": pd.DataFrame( columns=["plant_id_eia", "generator_id", "report_year", "capacity_mw"], data=[ (3, "1", "2017-01-01", 153.1), @@ -387,7 +387,7 @@ def test_resource_with_only_key_fields_harvests() -> None: (3989, "2018-01-01", "CO", "Boulder"), ], ), - "generation_eia923": pd.DataFrame( + "core_eia923__monthly_generation": pd.DataFrame( columns=["plant_id_eia", "generator_id", "report_month", "net_generation_mwh"], data=[ (3, "1", "2018-01-01", 10738.0), @@ -404,7 +404,7 @@ def test_resource_with_only_key_fields_harvests() -> None: (3, "1", "2018-12-01", -494.0), ], ), - "sales_eia861": pd.DataFrame( + "core_eia861__yearly_sales": pd.DataFrame( columns=["utility_id_eia", "report_year", "state", "county", "sales"], data=[ (3989, "2017-01-01", "CO", "El Paso", 127682), @@ -414,7 +414,7 @@ def test_resource_with_only_key_fields_harvests() -> None: (3989, "2018-01-01", "CO", "Boulder", 66666), ], ), - "boiler_generator_assn_eia860": pd.DataFrame( + "core_eia860__assn_boiler_generator": pd.DataFrame( columns=["plant_id_eia", "generator_id", "report_year", "boiler_id"], data=[ (3, "1", "2018-01-01", "1ST"), diff --git a/test/unit/helpers_test.py b/test/unit/helpers_test.py index ddf6d32029..6b99e15683 100644 --- a/test/unit/helpers_test.py +++ b/test/unit/helpers_test.py @@ -21,6 +21,7 @@ remove_leading_zeros_from_numeric_strings, zero_pad_numeric_string, ) +from pudl.metadata.helpers import _format_resource_name_cross_ref from pudl.output.sql.helpers import sql_asset_factory MONTHLY_GEN_FUEL = pd.DataFrame( @@ -626,9 +627,9 @@ def test_flatten_mix_types(): def test_cems_selection(): """Test CEMS asset selection remove cems assets.""" cems_selection = pudl.etl.create_non_cems_selection(pudl.etl.default_assets) - assert AssetKey("hourly_emissions_epacems") not in cems_selection.resolve( + assert AssetKey("core_epacems__hourly_emissions") not in cems_selection.resolve( pudl.etl.default_assets - ), "hourly_emissions_epacems or downstream asset present in selection." + ), "core_epacems__hourly_emissions or downstream asset present in selection." def test_sql_asset_factory_missing_file(): @@ -669,6 +670,19 @@ def test_convert_col_to_bool(df): ) +@pytest.mark.parametrize( + "test_input,expected", + [ + ("out_eia__yearly_generators", "out_eia__yearly_generators"), + ("_out_eia__yearly_generators", "i_out_eia__yearly_generators"), + ("__out_eia__yearly_generators", "i__out_eia__yearly_generators"), + ("", ""), + ], +) +def test_format_resource_name_cross_ref(test_input, expected): + assert _format_resource_name_cross_ref(test_input) == expected + + def test_diff_wide_tables(): # has 2020-2021 data for utils 1 and 2; fact 2 for utility 1 just never reported old = pd.DataFrame.from_records( diff --git a/test/unit/transform/ferc1_test.py b/test/unit/transform/ferc1_test.py index e1e40cf481..4bc1935278 100644 --- a/test/unit/transform/ferc1_test.py +++ b/test/unit/transform/ferc1_test.py @@ -265,7 +265,7 @@ def test_select_current_year_annual_records_duration_xbrl(): class FakeTransformer(Ferc1AbstractTableTransformer): # just need any table name here so that one method is callable - table_id = TableIdFerc1.FUEL_FERC1 + table_id = TableIdFerc1.STEAM_PLANTS_FUEL fake_transformer = FakeTransformer() df_out = fake_transformer.select_current_year_annual_records_duration_xbrl(df=df) @@ -296,7 +296,8 @@ def test_drop_duplicate_rows_dbf(): ) ) params = DropDuplicateRowsDbf( - table_name="balance_sheet_assets_ferc1", data_columns=["data_col1", "data_col2"] + table_name="core_ferc1__yearly_balance_sheet_assets_sched110", + data_columns=["data_col1", "data_col2"], ) df_out = drop_duplicate_rows_dbf(df, params=params).reset_index(drop=True) df_expected = pd.read_csv( @@ -504,7 +505,7 @@ def test_apply_xbrl_calculation_fixes(): class FakeTransformer(Ferc1AbstractTableTransformer): # just need any table name here so that one method is callable - table_id = TableIdFerc1.FUEL_FERC1 + table_id = TableIdFerc1.STEAM_PLANTS_FUEL calc_comps_fixed_out = FakeTransformer().apply_xbrl_calculation_fixes( calc_components=calc_comps_fix_test, calc_fixes=calc_fixes_test diff --git a/test/unit/transform/glue_test.py b/test/unit/transform/glue_test.py index 5be88015f7..7ff18ef227 100644 --- a/test/unit/transform/glue_test.py +++ b/test/unit/transform/glue_test.py @@ -76,11 +76,11 @@ def test_epacamd_eia_subplant_ids(): ) ) - epacamd_eia_subplant_ids_got = glue_assets.epacamd_eia_subplant_ids( - epacamd_eia_unique=epacamd_eia_test, - generators_eia860=generators_entity_eia_test, + epacamd_eia_subplant_ids_got = glue_assets.core_epa__assn_eia_epacamd_subplant_ids( + _core_epa__assn_eia_epacamd_unique=epacamd_eia_test, + core_eia860__scd_generators=generators_entity_eia_test, emissions_unit_ids_epacems=emissions_unit_ids_epacems_test, - boiler_generator_assn_eia860=boiler_generator_assn_eia860_test, + core_eia860__assn_boiler_generator=boiler_generator_assn_eia860_test, ) pd.testing.assert_frame_equal( diff --git a/test/validate/eia860_test.py b/test/validate/eia860_test.py index d5f81b9058..db8c87169b 100644 --- a/test/validate/eia860_test.py +++ b/test/validate/eia860_test.py @@ -189,7 +189,7 @@ def test_nuclear_units_are_generators(pudl_out_eia, live_dbs): # Drop all nuclear units that don't have a nuclear_unit_id. all_nuke_gf = ( pd.read_sql( - "generation_fuel_nuclear_eia923", + "core_eia923__monthly_generation_fuel_nuclear", pudl_out_eia.pudl_engine, columns=[ "plant_id_eia", diff --git a/test/validate/eia_test.py b/test/validate/eia_test.py index 3d9743e4dd..b94b7b4d83 100644 --- a/test/validate/eia_test.py +++ b/test/validate/eia_test.py @@ -108,72 +108,81 @@ def test_minmax_rows( @pytest.mark.parametrize( "df_name,unique_subset", [ - ("bf_eia923", Resource.from_id("boiler_fuel_eia923").schema.primary_key), + ( + "bf_eia923", + Resource.from_id("core_eia923__monthly_boiler_fuel").schema.primary_key, + ), ( "bga_eia860", - Resource.from_id("boiler_generator_assn_eia860").schema.primary_key, + Resource.from_id("core_eia860__assn_boiler_generator").schema.primary_key, ), ( "boil_eia860", - Resource.from_id("boilers_eia860").schema.primary_key, + Resource.from_id("core_eia860__scd_boilers").schema.primary_key, ), ( "gen_eia923", - Resource.from_id("generation_eia923").schema.primary_key, + Resource.from_id("core_eia923__monthly_generation").schema.primary_key, ), ( "gens_eia860", - Resource.from_id("generators_eia860").schema.primary_key, + Resource.from_id("core_eia860__scd_generators").schema.primary_key, ), ( "gf_eia923", - Resource.from_id( - "denorm_generation_fuel_combined_eia923" - ).schema.primary_key, + Resource.from_id("out_eia923__generation_fuel_combined").schema.primary_key, ), ( "own_eia860", - Resource.from_id("ownership_eia860").schema.primary_key, + Resource.from_id("core_eia860__scd_ownership").schema.primary_key, ), ( "plants_eia860", - Resource.from_id("plants_eia860").schema.primary_key, + Resource.from_id("core_eia860__scd_plants").schema.primary_key, ), ( "pu_eia860", - Resource.from_id("denorm_plants_utilities_eia").schema.primary_key, + Resource.from_id("_out_eia__plants_utilities").schema.primary_key, ), ( "utils_eia860", - Resource.from_id("utilities_eia860").schema.primary_key, + Resource.from_id("core_eia860__scd_utilities").schema.primary_key, ), ( "denorm_emissions_control_equipment_eia860", ( Resource.from_id( - "denorm_emissions_control_equipment_eia860" + "out_eia860__yearly_emissions_control_equipment" ).schema.primary_key ), ), ( "emissions_control_equipment_eia860", - (Resource.from_id("emissions_control_equipment_eia860").schema.primary_key), + ( + Resource.from_id( + "core_eia860__scd_emissions_control_equipment" + ).schema.primary_key + ), ), ( "boiler_emissions_control_equipment_assn_eia860", ( Resource.from_id( - "boiler_emissions_control_equipment_assn_eia860" + "core_eia860__assn_yearly_boiler_emissions_control_equipment" ).schema.primary_key ), ), ( "boiler_cooling_assn_eia860", - (Resource.from_id("boiler_cooling_assn_eia860").schema.primary_key), + (Resource.from_id("core_eia860__assn_boiler_cooling").schema.primary_key), ), ( "boiler_stack_flue_assn_eia860", - (Resource.from_id("boiler_stack_flue_assn_eia860").schema.primary_key), + ( + Resource.from_id( + "core_eia860__assn_boiler_stack_flue" + ).schema.primary_key + ), ), ], ) diff --git a/test/validate/ferc1_test.py b/test/validate/ferc1_test.py index de3f1e5d1b..434090e4e9 100644 --- a/test/validate/ferc1_test.py +++ b/test/validate/ferc1_test.py @@ -18,24 +18,24 @@ # unique record ID. But we should parameterize the has_unique_record_ids class # attributes in the FERC classes. non_unique_record_id_tables = [ - "plant_in_service_ferc1", - "purchased_power_ferc1", - "electric_energy_sources_ferc1", - "electric_energy_dispositions_ferc1", - "utility_plant_summary_ferc1", - "transmission_statistics_ferc1", - "balance_sheet_liabilities_ferc1", - "balance_sheet_assets_ferc1", - "income_statement_ferc1", - "depreciation_amortization_summary_ferc1", - "electric_plant_depreciation_changes_ferc1", - "electric_plant_depreciation_functional_ferc1", - "electric_operating_expenses_ferc1", - "cash_flow_ferc1", - "retained_earnings_ferc1", - "electric_operating_revenues_ferc1", - "other_regulatory_liabilities_ferc1", - "electricity_sales_by_rate_schedule_ferc1", + "core_ferc1__yearly_plant_in_service_sched204", + "core_ferc1__yearly_purchased_power_and_exchanges_sched326", + "core_ferc1__yearly_energy_sources_sched401", + "core_ferc1__yearly_energy_dispositions_sched401", + "core_ferc1__yearly_utility_plant_summary_sched200", + "core_ferc1__yearly_transmission_lines_sched422", + "core_ferc1__yearly_balance_sheet_liabilities_sched110", + "core_ferc1__yearly_balance_sheet_assets_sched110", + "core_ferc1__yearly_income_statements_sched114", + "core_ferc1__yearly_depreciation_summary_sched336", + "core_ferc1__yearly_depreciation_changes_sched219", + "core_ferc1__yearly_depreciation_by_function_sched219", + "core_ferc1__yearly_operating_expenses_sched320", + "core_ferc1__yearly_cash_flows_sched120", + "core_ferc1__yearly_retained_earnings_sched118", + "core_ferc1__yearly_operating_revenues_sched300", + "core_ferc1__yearly_other_regulatory_liabilities_sched278", + "core_ferc1__yearly_sales_by_rate_schedules_sched304", ] unique_record_tables = [ t diff --git a/test/validate/fuel_ferc1_test.py b/test/validate/fuel_ferc1_test.py index 646e6763a0..6739acf0f3 100644 --- a/test/validate/fuel_ferc1_test.py +++ b/test/validate/fuel_ferc1_test.py @@ -16,7 +16,9 @@ def test_fuel_ferc1_trivial(pudl_out_ferc1): """Test output routines for tables from FERC Form 1.""" logger.info("Compiling FERC Form 1 fuel table...") - fuel_tab = pd.read_sql("denorm_fuel_ferc1", pudl_out_ferc1.pudl_engine) + fuel_tab = pd.read_sql( + "out_ferc1__yearly_steam_plants_fuel_sched402", pudl_out_ferc1.pudl_engine + ) assert len(fuel_tab) > 0, "FERC Form 1 fuel table is empty." logger.info(f"{len(fuel_tab)} fuel records found") @@ -50,12 +52,16 @@ def test_fuel_ferc1_trivial(pudl_out_ferc1): ], ) def test_vs_bounds(pudl_out_ferc1, live_dbs, cases): - """Test distributions of reported plants_steam_ferc1 columns.""" + """Test distributions of reported core_ferc1__yearly_steam_plants_sched402 columns.""" if not live_dbs: pytest.skip("Data validation only works with a live PUDL DB.") for case in cases: pv.vs_bounds( - pd.read_sql("denorm_fuel_ferc1", pudl_out_ferc1.pudl_engine), **case + pd.read_sql( + "out_ferc1__yearly_steam_plants_fuel_sched402", + pudl_out_ferc1.pudl_engine, + ), + **case, ) @@ -64,4 +70,10 @@ def test_self_vs_historical(pudl_out_ferc1, live_dbs): if not live_dbs: pytest.skip("Data validation only works with a live PUDL DB.") for args in pv.fuel_ferc1_self: - pv.vs_self(pd.read_sql("denorm_fuel_ferc1", pudl_out_ferc1.pudl_engine), **args) + pv.vs_self( + pd.read_sql( + "out_ferc1__yearly_steam_plants_fuel_sched402", + pudl_out_ferc1.pudl_engine, + ), + **args, + ) diff --git a/test/validate/plants_steam_ferc1_test.py b/test/validate/plants_steam_ferc1_test.py index a1329ff84c..a95f6be0d1 100644 --- a/test/validate/plants_steam_ferc1_test.py +++ b/test/validate/plants_steam_ferc1_test.py @@ -17,22 +17,29 @@ @pytest.mark.parametrize( "cases", [ - pytest.param(pv.plants_steam_ferc1_capacity, id="capacity"), - pytest.param(pv.plants_steam_ferc1_expenses, id="expenses"), - pytest.param(pv.plants_steam_ferc1_capacity_ratios, id="capacity_ratios"), pytest.param( - pv.plants_steam_ferc1_connected_hours, + pv.core_ferc1__yearly_steam_plants_sched402_capacity, id="capacity" + ), + pytest.param( + pv.core_ferc1__yearly_steam_plants_sched402_expenses, id="expenses" + ), + pytest.param( + pv.core_ferc1__yearly_steam_plants_sched402_capacity_ratios, + id="capacity_ratios", + ), + pytest.param( + pv.core_ferc1__yearly_steam_plants_sched402_connected_hours, id="connected_hours", marks=pytest.mark.xfail(reason="FERC 1 data reporting errors."), ), ], ) def test_vs_bounds(pudl_out_ferc1, live_dbs, cases): - """Test distributions of reported plants_steam_ferc1 columns.""" + """Test distributions of reported core_ferc1__yearly_steam_plants_sched402 columns.""" if not live_dbs: pytest.skip("Data validation only works with a live PUDL DB.") validate_df = pd.read_sql( - "denorm_plants_steam_ferc1", pudl_out_ferc1.pudl_engine + "_out_ferc1__yearly_steam_plants_sched402", pudl_out_ferc1.pudl_engine ).assign( water_limited_ratio=lambda x: x.water_limited_capacity_mw / x.capacity_mw, not_water_limited_ratio=lambda x: x.not_water_limited_capacity_mw @@ -49,7 +56,7 @@ def test_self_vs_historical(pudl_out_ferc1, live_dbs): if not live_dbs: pytest.skip("Data validation only works with a live PUDL DB.") validate_df = pd.read_sql( - "denorm_plants_steam_ferc1", pudl_out_ferc1.pudl_engine + "_out_ferc1__yearly_steam_plants_sched402", pudl_out_ferc1.pudl_engine ).assign( water_limited_ratio=lambda x: x.water_limited_capacity_mw / x.capacity_mw, not_water_limited_ratio=lambda x: x.not_water_limited_capacity_mw @@ -57,7 +64,7 @@ def test_self_vs_historical(pudl_out_ferc1, live_dbs): peak_demand_ratio=lambda x: x.peak_demand_mw / x.capacity_mw, capability_ratio=lambda x: x.plant_capability_mw / x.capacity_mw, ) - for args in pv.plants_steam_ferc1_self: + for args in pv.core_ferc1__yearly_steam_plants_sched402_self: pudl.validate.vs_self(validate_df, **args) @@ -69,7 +76,9 @@ def test_dupe_years_in_plant_id_ferc1(pudl_out_ferc1): more than one record from a given year. Fail the test if we find such cases (which... we do, as of writing). """ - steam_df = pd.read_sql("denorm_plants_steam_ferc1", pudl_out_ferc1.pudl_engine) + steam_df = pd.read_sql( + "_out_ferc1__yearly_steam_plants_sched402", pudl_out_ferc1.pudl_engine + ) year_dupes = ( steam_df.groupby(["plant_id_ferc1", "report_year"])["utility_id_ferc1"] .count() @@ -97,7 +106,9 @@ def test_plant_id_clash(pudl_out_ferc1): only ever appear within a single PUDL Plant ID. Test this assertion and fail if it is untrue (as... we know it is right now). """ - steam_df = pd.read_sql("denorm_plants_steam_ferc1", pudl_out_ferc1.pudl_engine) + steam_df = pd.read_sql( + "_out_ferc1__yearly_steam_plants_sched402", pudl_out_ferc1.pudl_engine + ) bad_plant_ids_ferc1 = ( steam_df[["plant_id_pudl", "plant_id_ferc1"]] .drop_duplicates()