From 33bded0ac93e9c78afd1532d06526f5925d275f8 Mon Sep 17 00:00:00 2001 From: Florian Maurer Date: Wed, 23 Oct 2024 01:12:36 +0200 Subject: [PATCH] fix linting and error handling --- crawler/axxteq.py | 2 +- crawler/ecmwf.py | 2 +- crawler/enet.py | 2 +- crawler/regelleistung.py | 2 +- crawler/vea_industrial_load_profiles.py | 27 +++++++++++-------------- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/crawler/axxteq.py b/crawler/axxteq.py index cbed034..805232a 100644 --- a/crawler/axxteq.py +++ b/crawler/axxteq.py @@ -16,7 +16,7 @@ import pandas as pd from sqlalchemy import create_engine, text -from config import db_uri +from common.config import db_uri log = logging.getLogger(__name__) diff --git a/crawler/ecmwf.py b/crawler/ecmwf.py index 43f4dc9..e4080a3 100644 --- a/crawler/ecmwf.py +++ b/crawler/ecmwf.py @@ -17,7 +17,7 @@ from shapely.geometry import Point from sqlalchemy import create_engine, text -from config import db_uri +from common.config import db_uri """ Note that only requests with no more that 1000 items at a time are valid. diff --git a/crawler/enet.py b/crawler/enet.py index 9e5840a..ff48622 100644 --- a/crawler/enet.py +++ b/crawler/enet.py @@ -19,7 +19,7 @@ import pandas as pd from sqlalchemy import create_engine, text -from config import db_uri +from common.config import db_uri log = logging.getLogger("MaStR") log.setLevel(logging.INFO) diff --git a/crawler/regelleistung.py b/crawler/regelleistung.py index 4aeb6f8..dfc14b8 100644 --- a/crawler/regelleistung.py +++ b/crawler/regelleistung.py @@ -19,7 +19,7 @@ import sqlalchemy from sqlalchemy import create_engine, text -from .config import db_uri +from common.config import db_uri log = logging.getLogger("regelleistung") log.setLevel(logging.INFO) diff --git a/crawler/vea_industrial_load_profiles.py b/crawler/vea_industrial_load_profiles.py index 619502d..3447d46 100644 --- a/crawler/vea_industrial_load_profiles.py +++ b/crawler/vea_industrial_load_profiles.py @@ -16,7 +16,7 @@ "schema_name": "vea-industrial-load-profiles", "data_date": "2016-01-01", "data_source": "https://zenodo.org/records/13910298", - "license": "Creative Commons Attribution 4.0 International Public License", + "license": "CC-BY-4.0", "description": """The data consists of 5359 one-year quarterhourly industrial load profiles (2016, leap year, 35136 values). Each describes the electricity consumption of one industrial commercial site in Germany used for official accounting. Local electricity generation was excluded from the data as far as it could be discovered (no guarantee of completeness). @@ -194,11 +194,7 @@ def create_schema(): engine = create_engine(db_uri) with engine.begin() as conn: - query = text( - """ - CREATE SCHEMA IF NOT EXISTS "vea-industrial-load-profiles" - """ - ) + query = text('CREATE SCHEMA IF NOT EXISTS "vea-industrial-load-profiles"') conn.execute(query) log.info("Succesfully created schema") @@ -215,17 +211,18 @@ def convert_to_hypertable(relation_name: str): log.info("Trying to create hypertable") engine = create_engine(db_uri) - - with engine.begin() as conn: - query = text( - f"SELECT public.create_hypertable('{relation_name}', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE);" - ) - conn.execute(query) - - log.info("Succesfully create hypertable") + try: + with engine.begin() as conn: + query = text( + f"SELECT public.create_hypertable('{relation_name}', 'timestamp', if_not_exists => TRUE, migrate_data => TRUE);" + ) + conn.execute(query) + log.info("Succesfully create hypertable") + except Exception as e: + log.error(f"could not create hypertable: {e}") -def main(): +def main(db_uri): # request zip archive response = request_zip_archive()