diff --git a/config/zones/NL.yaml b/config/zones/NL.yaml index 975aa5092a..137ab1de9c 100644 --- a/config/zones/NL.yaml +++ b/config/zones/NL.yaml @@ -52,8 +52,8 @@ contributors: - corradio - PaulCornelissen - nessie2013 -disclaimer: The generation mix served by the source is incomplete. It is therefore - improved through an Electricity Maps estimation model. + - VIKTORVAV99 +disclaimer: Solar production is collected from NED.nl and can contain their own estimations. emissionFactors: direct: battery discharge: @@ -320,9 +320,9 @@ parsers: consumptionForecast: ENTSOE.fetch_consumption_forecast generationForecast: ENTSOE.fetch_generation_forecast price: ENTSOE.fetch_price - production: NL.fetch_production + production: NED.fetch_production productionCapacity: ENTSOE.fetch_production_capacity - productionPerModeForecast: ENTSOE.fetch_wind_solar_forecasts + productionPerModeForecast: NED.fetch_production_forecast price_displayed: false sources: EU-ETS, ENTSO-E 2021: diff --git a/electricitymap/contrib/lib/models/event_lists.py b/electricitymap/contrib/lib/models/event_lists.py index 9cdd1eb966..239f2bf168 100644 --- a/electricitymap/contrib/lib/models/event_lists.py +++ b/electricitymap/contrib/lib/models/event_lists.py @@ -303,20 +303,45 @@ def update_production_breakdowns( production_breakdowns: "ProductionBreakdownList", new_production_breakdowns: "ProductionBreakdownList", logger: Logger, + matching_timestamps_only: bool = False, ) -> "ProductionBreakdownList": - """Given a new batch of production breakdowns, update the existing ones.""" + """ + Given a new batch of production breakdowns, update the existing ones. + + Params: + - production_breakdowns: The existing production breakdowns to be updated. + - new_production_breakdowns: The new batch of production breakdowns. + - logger: The logger object used for logging information. + - matching_timestamps_only: Flag indicating whether to update only the events with matching timestamps from both the production breakdowns. + """ + if len(new_production_breakdowns) == 0: return production_breakdowns elif len(production_breakdowns) == 0: return new_production_breakdowns + updated_production_breakdowns = ProductionBreakdownList(logger) + + if matching_timestamps_only: + diff = abs(len(new_production_breakdowns) - len(production_breakdowns)) + logger.info( + f"Filtering production breakdowns to keep only the events where both the production breakdowns have matching datetimes, {diff} events where discarded." + ) + for new_event in new_production_breakdowns.events: if new_event.datetime in production_breakdowns: existing_event = production_breakdowns[new_event.datetime] updated_event = ProductionBreakdown._update(existing_event, new_event) - production_breakdowns[new_event.datetime] = updated_event - else: - production_breakdowns.append( + updated_production_breakdowns.append( + updated_event.zoneKey, + updated_event.datetime, + updated_event.source, + updated_event.production, + updated_event.storage, + updated_event.sourceType, + ) + elif matching_timestamps_only is False: + updated_production_breakdowns.append( new_event.zoneKey, new_event.datetime, new_event.source, @@ -325,7 +350,19 @@ def update_production_breakdowns( new_event.sourceType, ) - return production_breakdowns + if matching_timestamps_only is False: + for existing_event in production_breakdowns.events: + if existing_event.datetime not in new_production_breakdowns: + updated_production_breakdowns.append( + existing_event.zoneKey, + existing_event.datetime, + existing_event.source, + existing_event.production, + existing_event.storage, + existing_event.sourceType, + ) + + return updated_production_breakdowns @staticmethod def filter_only_zero_production( diff --git a/electricitymap/contrib/lib/models/events.py b/electricitymap/contrib/lib/models/events.py index 8dd672a373..de0361fcd3 100644 --- a/electricitymap/contrib/lib/models/events.py +++ b/electricitymap/contrib/lib/models/events.py @@ -670,20 +670,19 @@ def _update( raise ValueError( f"Cannot update events from different datetimes: {event.datetime} and {new_event.datetime}" ) - if event.source != new_event.source: - raise ValueError( - f"Cannot update events from different sources: {event.source} and {new_event.source}" - ) if event.sourceType != new_event.sourceType: raise ValueError( f"Cannot update events from different source types: {event.sourceType} and {new_event.sourceType}" ) production_mix = ProductionMix._update(event.production, new_event.production) storage_mix = StorageMix._update(event.storage, new_event.storage) + source = ", ".join( + set(event.source.split(", ")) | set(new_event.source.split(", ")) + ) return ProductionBreakdown( zoneKey=event.zoneKey, datetime=event.datetime, - source=event.source, + source=source, production=production_mix, storage=storage_mix, sourceType=event.sourceType, diff --git a/electricitymap/contrib/lib/tests/test_event_lists.py b/electricitymap/contrib/lib/tests/test_event_lists.py index 330ffdc7ab..2176b7c291 100644 --- a/electricitymap/contrib/lib/tests/test_event_lists.py +++ b/electricitymap/contrib/lib/tests/test_event_lists.py @@ -839,14 +839,20 @@ def test_update_production_with_different_source(self): zoneKey=ZoneKey("AT"), datetime=datetime(2023, 1, 1, tzinfo=timezone.utc), production=ProductionMix(wind=20, coal=20), - source="dont.trust.me", + source="trust.me.too", ) - self.assertRaises( - ValueError, - ProductionBreakdownList.update_production_breakdowns, - production_list1, - production_list2, - logging.Logger("test"), + updated_list = ProductionBreakdownList.update_production_breakdowns( + production_list1, production_list2, logging.Logger("test") + ) + assert len(updated_list.events) == 1 + assert updated_list.events[0].datetime == datetime( + 2023, 1, 1, tzinfo=timezone.utc + ) + assert updated_list.events[0].production is not None + assert updated_list.events[0].production.wind == 20 + assert updated_list.events[0].production.coal == 20 + assert updated_list.events[0].source == ", ".join( + set("trust.me, trust.me.too".split(", ")) ) def test_update_production_with_different_sourceType(self): diff --git a/parsers/NED.py b/parsers/NED.py new file mode 100644 index 0000000000..c042b6d8e1 --- /dev/null +++ b/parsers/NED.py @@ -0,0 +1,249 @@ +from datetime import datetime, timedelta, timezone +from enum import Enum +from logging import Logger, getLogger +from typing import Any + +import pandas as pd +import requests +from requests import Session + +from electricitymap.contrib.lib.models.event_lists import ProductionBreakdownList +from electricitymap.contrib.lib.models.events import ( + EventSourceType, + ProductionMix, +) +from electricitymap.contrib.lib.types import ZoneKey + +from .ENTSOE import ENTSOE_DOMAIN_MAPPINGS, WindAndSolarProductionForecastTypes +from .ENTSOE import parse_production as ENTSOE_parse_production +from .ENTSOE import query_production as ENTSOE_query_production +from .ENTSOE import ( + query_wind_solar_production_forecast as ENTSOE_query_wind_solar_production_forecast, +) +from .lib.exceptions import ParserException +from .lib.utils import get_token + +URL = "https://api.ned.nl/v1/utilizations" + +TYPE_MAPPING = { + 2: "solar", +} + + +class NedType(Enum): + SOLAR = 2 + + +class NedActivity(Enum): + PRODUCTION = 1 + CONSUMPTION = 2 + + +class NedGranularity(Enum): + TEN_MINUTES = 3 + FIFTEEN_MINUTES = 4 + HOURLY = 5 + DAILY = 6 + MONTHLY = 7 + YEARLY = 8 + + +class NedGranularityTimezone(Enum): + UTC = 0 + LOCAL = 1 + + +class NedClassification(Enum): + FORECAST = 1 + MEASURED = 2 + + +class NedPoint(Enum): + NETHERLANDS = 0 + + +# kWh to MWh with 3 decimal places +def _kwh_to_mw(kwh): + return round((kwh / 1000) * 4, 3) + + +# There seems to be a limitation of 144 items we can get in the response in the API at a time +# So we need to query each mode separately and then combine them +def call_api(target_datetime: datetime, forecast: bool = False): + params = { + "itemsPerPage": 192, + "point": NedPoint.NETHERLANDS.value, + "type[]": NedType.SOLAR.value, + "granularity": NedGranularity.FIFTEEN_MINUTES.value, + "granularitytimezone": NedGranularityTimezone.UTC.value, + "classification": NedClassification.FORECAST.value + if forecast + else NedClassification.MEASURED.value, + "activity": NedActivity.PRODUCTION.value, + "validfrom[before]": (target_datetime + timedelta(days=2 if forecast else 1)) + .date() + .isoformat(), + "validfrom[after]": (target_datetime - timedelta(days=0 if forecast else 1)) + .date() + .isoformat(), + } + headers = {"X-AUTH-TOKEN": get_token("NED_TOKEN"), "accept": "application/json"} + response = requests.get(URL, params=params, headers=headers) + if not response.ok: + raise ParserException( + parser="NED.py", + message=f"Failed to fetch NED data: {response.status_code}, err: {response.text}", + ) + return response.json() + + +def format_data( + json: Any, logger: Logger, forecast: bool = False +) -> ProductionBreakdownList: + df = pd.DataFrame(json) + df.drop( + columns=[ + "id", + "point", + "classification", + "activity", + "granularity", + "granularitytimezone", + "emission", + "emissionfactor", + "capacity", + "validto", + "lastupdate", + ], + inplace=True, + ) + + df = df.groupby(by="validfrom") + + formatted_production_data = ProductionBreakdownList(logger) + for _group_key, group_df in df: + data_dict = group_df.to_dict(orient="records") + mix = ProductionMix() + for data in data_dict: + clean_type = int(data["type"].split("/")[-1]) + if clean_type in TYPE_MAPPING: + mix.add_value( + TYPE_MAPPING[clean_type], + _kwh_to_mw(data["volume"]), + ) + else: + logger.warning(f"Unknown type: {clean_type}") + formatted_production_data.append( + zoneKey=ZoneKey("NL"), + datetime=group_df["validfrom"].iloc[0], + production=mix, + source="ned.nl", + sourceType=EventSourceType.forecasted + if forecast + else EventSourceType.measured, + ) + return formatted_production_data + + +def _get_entsoe_production_data( + zone_key: ZoneKey, + session: Session, + target_datetime: datetime, + logger: Logger, +) -> ProductionBreakdownList: + ENTSOE_raw_data = ENTSOE_query_production( + ENTSOE_DOMAIN_MAPPINGS[zone_key], session, target_datetime=target_datetime + ) + if ENTSOE_raw_data is None: + raise ParserException( + parser="NED.py", + message="Failed to fetch ENTSOE data", + zone_key=zone_key, + ) + ENTSOE_parsed_data = ENTSOE_parse_production( + ENTSOE_raw_data, zoneKey=zone_key, logger=logger + ) + return ENTSOE_parsed_data + + +def fetch_production( + zone_key: ZoneKey = ZoneKey("NL"), + session: Session | None = None, + target_datetime: datetime | None = None, + logger: Logger = getLogger(__name__), +) -> list: + session = session or Session() + target_datetime = target_datetime or datetime.now(timezone.utc) + json_data = call_api(target_datetime) + NED_data = format_data(json_data, logger) + ENTSOE_data = _get_entsoe_production_data( + zone_key, session, target_datetime, logger + ) + + combined_data = ProductionBreakdownList.update_production_breakdowns( + production_breakdowns=ENTSOE_data, + new_production_breakdowns=NED_data, + logger=logger, + matching_timestamps_only=True, + ) + + return combined_data.to_list() + + +def _get_entsoe_forecast_data( + zone_key: ZoneKey, + session: Session, + target_datetime: datetime, + logger: Logger, +) -> ProductionBreakdownList: + ENTSOE_raw_data_day_ahead = ENTSOE_query_wind_solar_production_forecast( + ENTSOE_DOMAIN_MAPPINGS[zone_key], + session, + data_type=WindAndSolarProductionForecastTypes.DAY_AHEAD, + target_datetime=target_datetime, + ) + ENTSOE_raw_data_intraday = ENTSOE_query_wind_solar_production_forecast( + ENTSOE_DOMAIN_MAPPINGS[zone_key], + session, + data_type=WindAndSolarProductionForecastTypes.INTRADAY, + target_datetime=target_datetime, + ) + if ENTSOE_raw_data_day_ahead is None or ENTSOE_raw_data_intraday is None: + raise ParserException( + parser="NED.py", + message="Failed to fetch ENTSOE data", + zone_key=zone_key, + ) + ENTSOE_parsed_data_day_ahead = ENTSOE_parse_production( + ENTSOE_raw_data_day_ahead, zoneKey=zone_key, logger=logger, forecasted=True + ) + ENTSOE_parsed_data_intraday = ENTSOE_parse_production( + ENTSOE_raw_data_intraday, zoneKey=zone_key, logger=logger, forecasted=True + ) + ENTSOE_updated_data = ProductionBreakdownList.update_production_breakdowns( + production_breakdowns=ENTSOE_parsed_data_day_ahead, + new_production_breakdowns=ENTSOE_parsed_data_intraday, + logger=logger, + ) + return ENTSOE_updated_data + + +def fetch_production_forecast( + zone_key: ZoneKey = ZoneKey("NL"), + session: Session | None = None, + target_datetime: datetime | None = None, + logger: Logger = getLogger(__name__), +) -> list: + session = session or Session() + target_datetime = target_datetime or datetime.now(timezone.utc) + json_data = call_api(target_datetime, forecast=True) + NED_data = format_data(json_data, logger, forecast=True) + ENTSOE_data = _get_entsoe_forecast_data(zone_key, session, target_datetime, logger) + + combined_data = ProductionBreakdownList.update_production_breakdowns( + production_breakdowns=ENTSOE_data, + new_production_breakdowns=NED_data, + logger=logger, + matching_timestamps_only=True, + ) + return combined_data.to_list()