Skip to content

Commit

Permalink
Updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@energinet-ajw
energinet-ajw committed Nov 5, 2024
1 parent 05734e8 commit e943316
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 5 deletions.
1 change: 0 additions & 1 deletion source/databricks/calculation_engine/package/datamigration/migration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
from .substitutions import get_substitutions

MIGRATION_SCRIPTS_FOLDER_PATH = "package.datamigration.migration_scripts"
UNUSED = "UNUSED" # Marks fields that were only used in the old hive implementation


def migrate_data_lake(
Expand Down
4 changes: 1 addition & 3 deletions source/databricks/calculation_engine/package/datamigration_hive/schema_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@
Schema(
name=paths.HiveOutputDatabase.DATABASE_NAME,
tables=[],
views=[
View(name=paths.HiveOutputDatabase.SUCCEEDED_ENERGY_RESULTS_V1_VIEW_NAME),
],
views=[],
),
Schema(
# Tables in this schema are externals and schemas are not defined in the SQL scripts.
Expand Down
24 changes: 23 additions & 1 deletion source/databricks/calculation_engine/package/infrastructure/paths.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,11 @@ class HiveOutputDatabase:
"""The folder in the storage account container"""

DATABASE_NAME = "wholesale_output"
SUCCEEDED_ENERGY_RESULTS_V1_VIEW_NAME = "succeeded_energy_results_v1"
#ENERGY_RESULT_TABLE_NAME = "energy_results"

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_integration_tests / flake8 and unit test 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test All Hive Scripts, datamigration/all_hive_scripts) / Test All Hive Scripts 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Current State Scripts, datamigration/current_state) / Current State Scripts 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Migration scripts, datamigration/migration_scripts) / Test Migration scripts 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Data products, features/data_products contracts) / Data products 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Unity Catalog, datamigration/unity_catalog) / Test Unity Catalog 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy (calculator_job), calculator_job/test_energy_calculation.py) / Energy (calculator_job) 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Entry point tests, entry_points/) / Entry point tests 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy logic, features/energy_calculation) / Energy logic 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Unit tests, calculation codelists common optimize_job databases infrastructure) / Unit tests 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale (calculator_job), calculator_job/test_wholesale_calculation.py) / Wholesale (calculator_job) 

block comment should start with '# '

Check failure on line 145 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale logic, features/wholesale_calculation) / Wholesale logic 

block comment should start with '# '
#WHOLESALE_RESULT_TABLE_NAME = "wholesale_results"

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_integration_tests / flake8 and unit test 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test All Hive Scripts, datamigration/all_hive_scripts) / Test All Hive Scripts 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Current State Scripts, datamigration/current_state) / Current State Scripts 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Migration scripts, datamigration/migration_scripts) / Test Migration scripts 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Data products, features/data_products contracts) / Data products 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Unity Catalog, datamigration/unity_catalog) / Test Unity Catalog 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy (calculator_job), calculator_job/test_energy_calculation.py) / Energy (calculator_job) 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Entry point tests, entry_points/) / Entry point tests 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy logic, features/energy_calculation) / Energy logic 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Unit tests, calculation codelists common optimize_job databases infrastructure) / Unit tests 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale (calculator_job), calculator_job/test_wholesale_calculation.py) / Wholesale (calculator_job) 

block comment should start with '# '

Check failure on line 146 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale logic, features/wholesale_calculation) / Wholesale logic 

block comment should start with '# '
#MONTHLY_AMOUNTS_TABLE_NAME = "monthly_amounts"

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_integration_tests / flake8 and unit test 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test All Hive Scripts, datamigration/all_hive_scripts) / Test All Hive Scripts 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Current State Scripts, datamigration/current_state) / Current State Scripts 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Migration scripts, datamigration/migration_scripts) / Test Migration scripts 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Data products, features/data_products contracts) / Data products 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Unity Catalog, datamigration/unity_catalog) / Test Unity Catalog 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy (calculator_job), calculator_job/test_energy_calculation.py) / Energy (calculator_job) 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Entry point tests, entry_points/) / Entry point tests 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy logic, features/energy_calculation) / Energy logic 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Unit tests, calculation codelists common optimize_job databases infrastructure) / Unit tests 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale (calculator_job), calculator_job/test_wholesale_calculation.py) / Wholesale (calculator_job) 

block comment should start with '# '

Check failure on line 147 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale logic, features/wholesale_calculation) / Wholesale logic 

block comment should start with '# '
#TOTAL_MONTHLY_AMOUNTS_TABLE_NAME = "total_monthly_amounts"

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_integration_tests / flake8 and unit test 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test All Hive Scripts, datamigration/all_hive_scripts) / Test All Hive Scripts 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Current State Scripts, datamigration/current_state) / Current State Scripts 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Migration scripts, datamigration/migration_scripts) / Test Migration scripts 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Data products, features/data_products contracts) / Data products 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Unity Catalog, datamigration/unity_catalog) / Test Unity Catalog 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy (calculator_job), calculator_job/test_energy_calculation.py) / Energy (calculator_job) 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Entry point tests, entry_points/) / Entry point tests 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy logic, features/energy_calculation) / Energy logic 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Unit tests, calculation codelists common optimize_job databases infrastructure) / Unit tests 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale (calculator_job), calculator_job/test_wholesale_calculation.py) / Wholesale (calculator_job) 

block comment should start with '# '

Check failure on line 148 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale logic, features/wholesale_calculation) / Wholesale logic 

block comment should start with '# '
#SUCCEEDED_ENERGY_RESULTS_V1_VIEW_NAME = "succeeded_energy_results_v1"

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_integration_tests / flake8 and unit test 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test All Hive Scripts, datamigration/all_hive_scripts) / Test All Hive Scripts 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Current State Scripts, datamigration/current_state) / Current State Scripts 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Migration scripts, datamigration/migration_scripts) / Test Migration scripts 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Data products, features/data_products contracts) / Data products 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / calculator_job_migrations_tests (Test Unity Catalog, datamigration/unity_catalog) / Test Unity Catalog 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy (calculator_job), calculator_job/test_energy_calculation.py) / Energy (calculator_job) 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Entry point tests, entry_points/) / Entry point tests 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Energy logic, features/energy_calculation) / Energy logic 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Unit tests, calculation codelists common optimize_job databases infrastructure) / Unit tests 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale (calculator_job), calculator_job/test_wholesale_calculation.py) / Wholesale (calculator_job) 

block comment should start with '# '

Check failure on line 149 in source/databricks/calculation_engine/package/infrastructure/paths.py

View workflow job for this annotation

GitHub Actions / ci_databricks / unit_tests (Wholesale logic, features/wholesale_calculation) / Wholesale logic 

block comment should start with '# '


class WholesaleBasisDataDatabase:
Expand Down Expand Up @@ -178,6 +182,24 @@ class HiveBasisDataDatabase:
"""The folder in the storage account container"""

DATABASE_NAME = "basis_data"
# To be removed when tables are deleted
# METERING_POINT_PERIODS_TABLE_NAME = "metering_point_periods"
# TIME_SERIES_POINTS_TABLE_NAME = "time_series_points"
# CHARGE_LINK_PERIODS_TABLE_NAME = "charge_link_periods"
# CHARGE_PRICE_INFORMATION_PERIODS_TABLE_NAME = "charge_price_information_periods"
# CHARGE_PRICE_POINTS_TABLE_NAME = "charge_price_points"
# GRID_LOSS_METERING_POINTS_TABLE_NAME = "grid_loss_metering_points"
# CALCULATIONS_TABLE_NAME = "calculations"
#
# TABLE_NAMES = [
# CALCULATIONS_TABLE_NAME,
# METERING_POINT_PERIODS_TABLE_NAME,
# TIME_SERIES_POINTS_TABLE_NAME,
# CHARGE_LINK_PERIODS_TABLE_NAME,
# CHARGE_PRICE_INFORMATION_PERIODS_TABLE_NAME,
# CHARGE_PRICE_POINTS_TABLE_NAME,
# GRID_LOSS_METERING_POINTS_TABLE_NAME,
# ]


# Hive
Expand Down

0 comments on commit e943316

Please sign in to comment.