diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 00000000..90814ea0 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,75 @@ +# ------------------------------------------------------------------ +# This allow to run the S2 CEM from your local FlexMeasures Client code in a docker compose stack. +# Assuming you have flexmeasures the repo next to your flexmeasures-client repo, +# run this from the flexmeasures folder (which contains the Dockerfile): +# docker compose \ +# -f docker-compose.yml \ +# -f ../flexmeasures-client/docker-compose.override.yml \ +# up +# ------------------------------------------------------------------ + +services: + dev-db: + ports: + - "5433:5432" + queue-db: + ports: + - "6380:6379" + server: + volumes: + # A place for config and plugin code, and custom requirements.txt + # The 1st mount point is for running the FlexMeasures CLI, the 2nd for gunicorn + # We use :rw so flexmeasures CLI commands can write log files + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ./flexmeasures-instance/:/app/instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + command: + - | + pip install --break-system-packages -e /app + pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt + pip install timely-beliefs -U --break-system-packages + flexmeasures db upgrade + if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then + flexmeasures add toy-account --name 'Docker Toy Account' + fi + gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 2 --threads 4 wsgi:application + worker: + volumes: + # a place for config and plugin code, and custom requirements.txt + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + - ./flexmeasures-instance/:/app/instance/:rw + command: + - | + pip install --break-system-packages -e /app + pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt + pip install timely-beliefs -U --break-system-packages + flexmeasures jobs run-worker --name flexmeasures-worker --queue ingestion\|scheduling\|forecasting + cem: + build: + context: . + dockerfile: Dockerfile + depends_on: + - server + restart: always + ports: + - "8080:8080" # aiohttp default; adjust if you change it + environment: + # Optional, but useful if you later make this configurable + FLEXMEASURES_BASE_URL: http://server:5000 + FLEXMEASURES_USER: toy-user@flexmeasures.io + FLEXMEASURES_PASSWORD: toy-password + LOGGING_LEVEL: DEBUG + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLEXMEASURES_CLIENT: "0.0.0" + volumes: + # If flexmeasures_client lives in your repo and you want live edits + - ../flexmeasures-client:/app/flexmeasures-client:rw + entrypoint: ["/bin/sh", "-c"] + command: + - | + # pip install --break-system-packages --no-cache-dir "git+https://github.com/FlexMeasures/flexmeasures-client.git@main#egg=flexmeasures-client[s2]" + pip install --break-system-packages -e /app/flexmeasures-client[s2] + pip install --break-system-packages aiohttp + pip install --break-system-packages pytz + pip install --break-system-packages s2-python==0.8.2 + python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py diff --git a/docs/CEM.rst b/docs/CEM.rst index ace53690..3defb180 100644 --- a/docs/CEM.rst +++ b/docs/CEM.rst @@ -54,12 +54,39 @@ Then point your Resource Managers (RMs) to ``http://localhost:8080/ws`` and run: uv run src/flexmeasures_client/s2/script/websockets_server.py +We also included a ``docker-compose.override.yaml`` that can be used to set up the CEM including the FlexMeasures server, creating a fully self-hosted HEMS. +Assuming your ``flexmeasures`` and ``flexmeasures-client`` repo folders are located side by side, run this from your flexmeasures folder: + +.. code-block:: bash + + docker compose \ + -f docker-compose.yml \ + -f ../flexmeasures-client/docker-compose.override.yml \ + up + + +This creates the following containers for the CEM: + +- a WebSocket server (FlexMeasures Client) +- web and worker servers (FlexMeasures) +- a database server (Postgres) +- a queue server (Redis) +- a mail server (MailHog) + To test, run the included example RM: .. code-block:: bash uv run src/flexmeasures_client/s2/script/websockets_client.py +For full access via the UI, create an admin user for the Docker Toy Account (here, we assume it has ID 1): + +.. code-block:: bash + + docker exec -it flexmeasures-server-1 bash + flexmeasures show accounts + flexmeasures add user --roles admin --account 1 --email --username + Disclaimer ========== diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 1c8fe259..bbe115d2 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -337,6 +337,7 @@ async def request_once( f"FlexMeasures server version changed from {self.server_version} to {header_version}." ) self.server_version = header_version + self.server_version = "0.33.0" # temp override until v0.33.0 is released polling_step, reauth_once, url = await check_response( self, response, polling_step, reauth_once, url @@ -759,7 +760,7 @@ async def get_asset( ) -> dict: """Fetch a single asset. - :param asset_id: ID of the asset to fetch + :param asset_id: ID of the asset to fetch. :param parse_json_fields: If True, parse JSON string fields (attributes, flex_context, flex_model) into Python dicts. If False, leave them as JSON strings for backward compatibility. If None (default), uses old behavior (no parsing) with a deprecation warning. @@ -1027,7 +1028,7 @@ async def get_sensor( ) -> dict: """Get a single sensor. - :param sensor_id: ID of the sensor to fetch + :param sensor_id: ID of the sensor to fetch. :param parse_json_fields: If True, parse JSON string fields (attributes) into Python dicts. If False, leave them as JSON strings for backward compatibility. If None (default), uses old behavior (no parsing) with a deprecation warning. @@ -1120,9 +1121,9 @@ async def add_asset( self, name: str, account_id: int, - latitude: float, - longitude: float, generic_asset_type_id: int, + latitude: float = None, + longitude: float = None, parent_asset_id: int | None = None, sensors_to_show: list | None = None, flex_context: dict | None = None, @@ -1151,10 +1152,12 @@ async def add_asset( asset = dict( name=name, account_id=account_id, - latitude=latitude, - longitude=longitude, generic_asset_type_id=generic_asset_type_id, ) + if latitude is not None: + asset["latitude"] = str(latitude) + if longitude is not None: + asset["longitude"] = str(longitude) if parent_asset_id: asset["parent_asset_id"] = parent_asset_id if sensors_to_show: @@ -1177,9 +1180,17 @@ async def add_asset( ) return new_asset - async def update_asset(self, asset_id: int, updates: dict) -> dict: + async def update_asset( + self, asset_id: int, updates: dict, parse_json_fields: bool | None = None + ) -> dict: """Patch an asset. + :param asset_id: ID of the asset to update. + :param updates: Dictionary with the fields to update. + :param parse_json_fields: If True, parse JSON string fields (attributes) into Python dicts. + If False, leave them as JSON strings for backward compatibility. + If None (default), uses old behavior (no parsing) with a deprecation warning. + Default will change to True in a future version. :returns: asset as dictionary, for example: { 'account_id': 2, @@ -1233,6 +1244,21 @@ async def update_asset(self, asset_id: int, updates: dict) -> dict: raise ContentTypeError( f"Expected an asset dictionary, but got {type(updated_asset)}", ) + + if parse_json_fields is None: + warnings.warn( + "The default behavior of update_asset() will change in a future version. " + "JSON fields (attributes) will be automatically parsed into Python dicts. " + "To opt into the new behavior now, pass parse_json_fields=True. " + "To silence this warning and keep the old behavior, pass parse_json_fields=False explicitly.", + FutureWarning, + stacklevel=2, + ) + parse_json_fields = False + + if parse_json_fields: + _parse_asset_json_fields(updated_asset) + return updated_asset async def delete_asset(self, asset_id: int, confirm_first: bool = True): @@ -1251,9 +1277,17 @@ async def delete_asset(self, asset_id: int, confirm_first: bool = True): _, status = await self.request(uri=uri, method="DELETE") check_for_status(status, 204) - async def update_sensor(self, sensor_id: int, updates: dict) -> dict: + async def update_sensor( + self, sensor_id: int, updates: dict, parse_json_fields: bool | None = None + ) -> dict: """Patch a sensor. + :param sensor_id: ID of the sensor to update. + :param updates: Dictionary with the fields to update. + :param parse_json_fields: If True, parse JSON string fields (attributes) into Python dicts. + If False, leave them as JSON strings for backward compatibility. + If None (default), uses old behavior (no parsing) with a deprecation warning. + Default will change to True in a future version. :returns: sensor as dictionary, for example: { 'attributes': '{}', @@ -1281,6 +1315,21 @@ async def update_sensor(self, sensor_id: int, updates: dict) -> dict: raise ContentTypeError( f"Expected a sensor dictionary, but got {type(updated_sensor)}", ) + + if parse_json_fields is None: + warnings.warn( + "The default behavior of update_sensor() will change in a future version. " + "JSON fields (attributes) will be automatically parsed into Python dicts. " + "To opt into the new behavior now, pass parse_json_fields=True. " + "To silence this warning and keep the old behavior, pass parse_json_fields=False explicitly.", + FutureWarning, + stacklevel=2, + ) + parse_json_fields = False + + if parse_json_fields: + _parse_sensor_json_fields(updated_sensor) + return updated_sensor async def delete_sensor(self, sensor_id: int, confirm_first: bool = True): @@ -1354,6 +1403,7 @@ async def trigger_schedule( ) self._sensor_asset_id_cache[sensor_id] = sensor["generic_asset_id"] asset_id = self._sensor_asset_id_cache[sensor_id] + flex_model["sensor"] = sensor_id # Move sensor ID into the flex-model if flex_model is None: diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 7e83cd8b..d78c9dd9 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -1,9 +1,9 @@ from __future__ import annotations +import asyncio import json import logging import math -from asyncio import Queue from collections import defaultdict from datetime import datetime, timedelta from logging import Logger @@ -34,7 +34,9 @@ from flexmeasures_client.client import FlexMeasuresClient from flexmeasures_client.s2 import Handler, register from flexmeasures_client.s2.control_types import ControlTypeHandler +from flexmeasures_client.s2.config_utils import configure_site from flexmeasures_client.s2.utils import ( + ControlContext, get_latest_compatible_version, get_reception_status, get_unique_id, @@ -49,7 +51,6 @@ class CEM(Handler): _resource_manager_details: ResourceManagerDetails _control_types_handlers: Dict[ControlType | None, ControlTypeHandler] - _control_type = None _is_closed = True _default_control_type: ControlType | None @@ -58,7 +59,7 @@ class CEM(Handler): ] # maps the CommodityQuantity power measurement sensors to FM sensor IDs _fm_client: FlexMeasuresClient - _sending_queue: Queue[pydantic.BaseModel] + _sending_queue: asyncio.Queue[tuple[pydantic.BaseModel, asyncio.Future]] _timers: dict[str, datetime] _datastore: dict @@ -81,10 +82,15 @@ def __init__( """ Customer Energy Manager (CEM) """ + # Initialize per-instance control context and handler build tasks BEFORE calling super().__init__() + # because parent's __init__ calls discover() which accesses control_type property + self._control = ControlContext() + self._handler_build_tasks: dict[ControlType, asyncio.Task] = {} + super(CEM, self).__init__() self._fm_client = fm_client - self._sending_queue = Queue() + self._sending_queue = asyncio.Queue() self._power_sensors = dict() self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() @@ -141,7 +147,7 @@ def is_closed(self): @property def control_type(self): - return self._control_type + return self._control.control_type def register_control_type(self, control_type_handler: ControlTypeHandler): """ @@ -158,8 +164,8 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): # add fm_client to control_type handler control_type_handler._fm_client = self._fm_client - # add sending queue - control_type_handler._sending_queue = self._sending_queue + # add send_message method so the handler can send messages + control_type_handler.send_message = self.send_message # Add logger control_type_handler._logger = self._logger @@ -169,6 +175,9 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): control_type_handler ) + # Mark handler as ready once registered + self._control.handler_ready[control_type_handler._control_type] = True + async def handle_message(self, message: Dict | pydantic.BaseModel | str): """ This method handles the incoming messages to the CEM and routes them to their custom handler. @@ -184,22 +193,32 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): if isinstance(message, str): message = json.loads(message) - self._logger.debug(f"Received: {message}") + # Detect wrapper + if isinstance(message, dict) and "message" in message and "metadata" in message: + metadata = message["metadata"] + message = message["message"] + self._logger.debug("Received wrapped message") + self._logger.debug(f"Received message: {message}") + self._logger.debug(f"Received metadata: {metadata}") + if "dt" in metadata: + for control_type in self._control_types_handlers.values(): + control_type.now = lambda: metadata["dt"] # type: ignore + self.now = lambda: metadata["dt"] # type: ignore + else: + self._logger.debug(f"Received: {message}") # try to handle the message with the control_type handle + ct = self._control.control_type + handler = self._control_types_handlers.get(ct) + ready = self._control.handler_ready.get(ct, False) + if ( - self._control_type is not None - and ( - self._control_type - not in [ControlType.NO_SELECTION, ControlType.NOT_CONTROLABLE] - ) - and self._control_types_handlers[self._control_type].supports_message( - message - ) + handler is not None + and ready + and ct not in [ControlType.NO_SELECTION, ControlType.NOT_CONTROLABLE] + and handler.supports_message(message) ): - response = await self._control_types_handlers[ - self._control_type - ].handle_message(message) + response = await handler.handle_message(message) else: if self.supports_message(message): response = await super().handle_message( @@ -215,24 +234,33 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): ) if response is not None: - await self._sending_queue.put(response) + await self.send_message(response) def update_control_type(self, control_type: ControlType): """ Callback function that is triggered when we receive a confirmation that the message has been received. """ - self._control_type = control_type + self._control.control_type = control_type - async def get_message(self) -> str: + async def get_message(self) -> tuple[str, asyncio.Future]: """Call this function to get the messages to be sent to the RM Returns: str: message in JSON format """ - message = await self._sending_queue.get() - return message.model_dump(mode="json") + item = await self._sending_queue.get() + + if not isinstance(item, tuple) or len(item) != 2: + raise RuntimeError( + "Invalid item in sending queue. All messages must go through send_message() rather than _sending_queue.put()." + ) + + message, fut = item + message = message.model_dump(mode="json") + + return message, fut async def activate_control_type( self, control_type: ControlType @@ -242,7 +270,7 @@ async def activate_control_type( """ # check if it's trying to activate the current control_type - if control_type == self._control_type: + if control_type == self._control.control_type: self._logger.warning(f"RM is already in `{control_type}` control type.") return None @@ -252,14 +280,14 @@ async def activate_control_type( return None # RM initialization succeeded - if self._control_type is not None: + if self._control.control_type is not None: message_id = get_unique_id() # the callback `update_control_type` will be called upon arrival of a # ReceptionStatus message with status = ReceptionStatusValues.OK # register callback in CEM handler - if self._control_type in [ + if self._control.control_type in [ ControlType.NOT_CONTROLABLE, ControlType.NO_SELECTION, ]: @@ -268,12 +296,11 @@ async def activate_control_type( ) else: # register callback in control mode handler self._control_types_handlers[ - self._control_type + self._control.control_type ].register_success_callbacks( message_id, self.update_control_type, control_type=control_type ) - - await self._sending_queue.put( + await self.send_message( SelectControlType(message_id=message_id, control_type=control_type) ) return None @@ -300,11 +327,21 @@ async def handle_handshake(self, message: Handshake): async def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message + # schedule map_resource_to_asset to run soon concurrently + task = asyncio.create_task(self.map_resource_to_asset(message)) + self._handler_build_tasks[ControlType.FILL_RATE_BASED_CONTROL] = task + # self.background_tasks.add( + # task + # ) # important to avoid a task disappearing mid-execution. + # task.add_done_callback(self.background_tasks.discard) + + # await self.map_resource_to_asset(message) + if ( - not self._control_type + not self._control.control_type ): # initializing. TODO: check if sending resource_manager_details # resets control type - self._control_type = ControlType.NO_SELECTION + self._control.control_type = ControlType.NO_SELECTION # Activate default control type if defined if self._default_control_type: @@ -312,6 +349,66 @@ async def handle_resource_manager_details(self, message: ResourceManagerDetails) return get_reception_status(message) + async def map_resource_to_asset(self, message): + """Map S2 resource to FM asset. + + - Creates a new asset if the resource ID does not yet exist in FlexMeasures. + - Updates the existing asset if the resource details changed. + - Updates the control type for the resource. + """ + assets = await self._fm_client.get_assets() + asset = None + for ast in assets: + if ast["external_id"] == message.resource_id: + asset = ast + if asset is None: + account = await self._fm_client.get_account() + asset = await self._fm_client.add_asset( + name=message.name, + account_id=account["id"], + generic_asset_type_id=1, + # parent_asset_id=self._asset_id, + attributes=json.loads(message.to_json()), + ) + if asset["name"] != message.name: + await self._fm_client.update_asset( + asset_id=asset["id"], updates={"name": message.name} + ) + if asset["attributes"] != message.to_json(): + await self._fm_client.update_asset( + asset_id=asset["id"], + updates={"attributes": json.loads(message.to_json())}, + ) + + # Reconfigure site + ( + price_sensor, + production_price_sensor, + power_sensor, + soc_sensor, + rm_discharge_sensor, + soc_minima_sensor, + soc_maxima_sensor, + usage_forecast_sensor, + leakage_behaviour_sensor, + charging_efficiency_sensor, + ) = await configure_site(message.name, self._fm_client) + from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple + + frbc = FRBCSimple( + power_sensor_id=power_sensor["id"], + price_sensor_id=price_sensor["id"], + production_price_sensor_id=production_price_sensor["id"], + soc_sensor_id=soc_sensor["id"], + rm_discharge_sensor_id=rm_discharge_sensor["id"], + soc_minima_sensor_id=soc_minima_sensor["id"], + soc_maxima_sensor_id=soc_maxima_sensor["id"], + usage_forecast_sensor_id=usage_forecast_sensor["id"], + leakage_behaviour_sensor_id=leakage_behaviour_sensor["id"], + charging_efficiency_sensor_id=charging_efficiency_sensor["id"], + ) + self.register_control_type(frbc) + @register(PowerMeasurement) async def handle_power_measurement(self, message: PowerMeasurement): @@ -418,8 +515,10 @@ def handle_revoke_object(self, message: RevokeObject): return get_reception_status(message, ReceptionStatusValues.OK) async def send_message(self, message): + loop = asyncio.get_running_loop() + fut = loop.create_future() self._logger.debug(f"Sent: {message}") - await self._sending_queue.put(message) + await self._sending_queue.put((message, fut)) def get_commodity_unit(commodity_quantity) -> str: diff --git a/src/flexmeasures_client/s2/config_utils.py b/src/flexmeasures_client/s2/config_utils.py new file mode 100644 index 00000000..6edff4a8 --- /dev/null +++ b/src/flexmeasures_client/s2/config_utils.py @@ -0,0 +1,220 @@ +import asyncio +import logging +import os +from datetime import datetime +from zoneinfo import ZoneInfo + +from flexmeasures_client.client import FlexMeasuresClient + +log_level = os.getenv("LOGGING_LEVEL", "WARNING").upper() +logging.basicConfig( + level=log_level, + format="[CEM][%(asctime)s] %(levelname)s: %(name)s | %(message)s", +) +LOGGER = logging.getLogger(__name__) + + +async def configure_site( + site_name: str, fm_client: FlexMeasuresClient +) -> tuple[dict, dict, dict, dict, dict, dict, dict, dict, dict, dict]: + account = await fm_client.get_account() + assets = await fm_client.get_assets(parse_json_fields=True) + + site_asset: dict | None = None + for asset in assets: + if asset["name"] == site_name: + site_asset = asset + break + + site_asset_specs = dict( + latitude=0, + longitude=0, + generic_asset_type_id=6, # Building asset type + flex_model={ + "power-capacity": f"{3 * 25 * 230} VA", + }, + ) + + if not site_asset: + site_asset = await fm_client.add_asset( + name=site_name, account_id=account["id"], **site_asset_specs + ) + # Update site asset with the latest specs + await fm_client.update_asset(site_asset["id"], site_asset_specs) + + sensors = site_asset.get("sensors", []) + price_sensor = None + production_price_sensor = None + power_sensor = None + soc_sensor = None + rm_discharge_sensor = None + soc_minima_sensor = None + soc_maxima_sensor = None + usage_forecast_sensor = None + leakage_behaviour_sensor = None + charging_efficiency_sensor = None + for sensor in sensors: + if sensor["name"] == "price": + price_sensor = sensor + if sensor["name"] == "production price": + production_price_sensor = sensor + elif sensor["name"] == "power": + power_sensor = sensor + elif sensor["name"] == "state of charge": + soc_sensor = sensor + elif sensor["name"] == "RM discharge": + rm_discharge_sensor = sensor + elif sensor["name"] == "soc-minima": + soc_minima_sensor = sensor + elif sensor["name"] == "soc-maxima": + soc_maxima_sensor = sensor + elif sensor["name"] == "usage-forecast": + usage_forecast_sensor = sensor + elif sensor["name"] == "leakage-behaviour": + leakage_behaviour_sensor = sensor + elif sensor["name"] == "charging-efficiency": + charging_efficiency_sensor = sensor + + if price_sensor is None: + price_sensor = await fm_client.add_sensor( + name="price", + event_resolution="PT15M", + unit="EUR/kWh", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + if production_price_sensor is None: + production_price_sensor = await fm_client.add_sensor( + name="production price", + event_resolution="PT15M", + unit="EUR/kWh", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + + # Continue immediately without awaiting + LOGGER.debug("Posting 3 days of prices in a background task..") + start_of_today = ( + datetime.now(ZoneInfo("Europe/Amsterdam")) + .replace(hour=0, minute=0, second=0, microsecond=0) + .isoformat() + ) + asyncio.create_task( + fm_client.post_sensor_data( + sensor_id=price_sensor["id"], + start=start_of_today, + prior="2026-01-01T00:00+01", # 2026-01-01T00:00+01 + duration="P3D", # P1M + values=[0.3], + unit="EUR/kWh", + ) + ) + asyncio.create_task( + fm_client.post_sensor_data( + sensor_id=production_price_sensor["id"], + start=start_of_today, + prior="2026-01-01T00:00+01", # 2026-01-01T00:00+01 + duration="P3D", # P1M + values=[0.2], + unit="EUR/kWh", + ) + ) + if power_sensor is None: + power_sensor = await fm_client.add_sensor( + name="power", + event_resolution="PT15M", + unit="kW", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + attributes={"consumption_is_positive": True}, + ) + if soc_sensor is None: + soc_sensor = await fm_client.add_sensor( + name="state of charge", + event_resolution="PT0M", + unit="kWh", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + if rm_discharge_sensor is None: + rm_discharge_sensor = await fm_client.add_sensor( + name="RM discharge", + event_resolution="PT15M", + unit="dimensionless", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + if soc_minima_sensor is None: + soc_minima_sensor = await fm_client.add_sensor( + name="soc-minima", + event_resolution="PT15M", + unit="kWh", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + if soc_maxima_sensor is None: + soc_maxima_sensor = await fm_client.add_sensor( + name="soc-maxima", + event_resolution="PT15M", + unit="kWh", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + if usage_forecast_sensor is None: + usage_forecast_sensor = await fm_client.add_sensor( + name="usage-forecast", + event_resolution="PT15M", + unit="kW", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + if leakage_behaviour_sensor is None: + leakage_behaviour_sensor = await fm_client.add_sensor( + name="leakage-behaviour", + event_resolution="PT15M", + unit="%", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + if charging_efficiency_sensor is None: + charging_efficiency_sensor = await fm_client.add_sensor( + name="charging-efficiency", + event_resolution="PT15M", + unit="%", + generic_asset_id=site_asset["id"], + timezone="Europe/Amsterdam", + ) + sensors_to_show = [ + { + "title": "State of charge", + "sensors": [ + soc_minima_sensor["id"], + soc_maxima_sensor["id"], + soc_sensor["id"], + ], + }, + { + "title": "Prices", + "sensors": [price_sensor["id"], production_price_sensor["id"]], + }, + { + "title": "Power", + "sensors": [power_sensor["id"]], + }, + ] + await fm_client.update_asset( + asset_id=site_asset["id"], + updates=dict(sensors_to_show=sensors_to_show), + ) + return ( + price_sensor, + production_price_sensor, + power_sensor, + soc_sensor, + rm_discharge_sensor, + soc_minima_sensor, + soc_maxima_sensor, + usage_forecast_sensor, + leakage_behaviour_sensor, + charging_efficiency_sensor, + ) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 2b4ccf15..de03affd 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -77,8 +77,107 @@ def handle_system_description( task ) # important to avoid a task disappearing mid-execution. task.add_done_callback(self.background_tasks.discard) + + # schedule send_conversion_efficiencies to run soon concurrently + task = asyncio.create_task(self.send_conversion_efficiencies(message)) + self.background_tasks.add(task) + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) + def _get_operation_mode_efficiency_sensor_map( + self, system_description: FRBCSystemDescription + ) -> dict[str, int]: + """ + Get a mapping of operation mode IDs to efficiency sensor IDs. + + Subclasses can override this method to provide operation mode to efficiency + sensor mappings. Return an empty dict if there are no efficiency sensors. + + Args: + system_description: The system description containing operation mode details. + + Returns: + A dictionary mapping operation mode IDs to efficiency sensor IDs. + Empty dict ({}) if there are no efficiency sensors (default). + """ + return {} + + async def send_conversion_efficiencies( + self, system_description: FRBCSystemDescription + ): + """ + Send conversion efficiencies to FlexMeasures for operation modes. + + This method sends efficiency values for each operation mode that has + an associated efficiency sensor. Subclasses should override + _get_operation_mode_efficiency_sensor_map() to define which operation modes + have efficiency sensors. + + Args: + system_description: The system description containing actuator details. + """ + efficiency_map = self._get_operation_mode_efficiency_sensor_map( + system_description + ) + if not efficiency_map: + # No efficiency sensors defined + return + + try: + from datetime import datetime + from datetime import timedelta + + start = system_description.valid_from + actuator = system_description.actuators[0] + + start_time = start.replace( + minute=(start.minute // 15) * 15, second=0, microsecond=0 + ) + + # Use a default conversion efficiency duration if not set by subclass + duration = getattr(self, "_conversion_efficiency_duration", timedelta(hours=99)) + if isinstance(duration, str): + # If duration is a string like "PT99H", use default timedelta + duration = timedelta(hours=99) + + for operation_mode in actuator.operation_modes: + sensor_id = efficiency_map.get(operation_mode.id) + if sensor_id is None: + # Skip operation modes without an efficiency sensor + continue + + # Calculate efficiency from the last element (characteristic endpoint) + try: + fill_level_scale = getattr(self, "_fill_level_scale", 1.0) + efficiency = ( + 1 + * operation_mode.elements[-1].fill_rate.end_of_range + * fill_level_scale + / (operation_mode.elements[-1].power_ranges[0].end_of_range) + ) + except (IndexError, AttributeError, ZeroDivisionError) as e: + self._logger.debug( + f"Could not calculate efficiency for operation mode {operation_mode.id}: {e}" + ) + continue + + try: + await self._fm_client.post_sensor_data( + sensor_id=sensor_id, + start=start_time, + prior=self.now(), + values=[efficiency], + unit="dimensionless", + duration=duration, + ) + except Exception as e: + self._logger.debug( + f"Error posting efficiency data for sensor {sensor_id}: {e}" + ) + except Exception as e: + self._logger.debug(f"Error sending conversion efficiencies: {e}") + @register(FRBCUsageForecast) def handle_usage_forecast(self, message: FRBCUsageForecast) -> pydantic.BaseModel: message_id = str(message.message_id) @@ -196,4 +295,4 @@ async def trigger_schedule(self, system_description_id: str): ) # put instruction into the sending queue - await self._sending_queue.put(instruction) + await self.send_message(instruction) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index eafdaa1b..44395526 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -6,11 +6,16 @@ from datetime import datetime, timedelta from zoneinfo import ZoneInfo +import pandas as pd + try: from s2python.frbc import ( FRBCActuatorStatus, + FRBCFillLevelTargetProfile, + FRBCLeakageBehaviour, FRBCStorageStatus, FRBCSystemDescription, + FRBCUsageForecast, ) except ImportError: raise ImportError( @@ -24,15 +29,27 @@ fm_schedule_to_instructions, get_soc_min_max, ) +from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, + translate_fill_level_target_profile, + translate_usage_forecast_to_fm, +) class FRBCSimple(FRBC): _power_sensor_id: int _price_sensor_id: int + _production_price_sensor_id: int _soc_sensor_id: int _rm_discharge_sensor_id: int + _soc_minima_sensor_id: int + _soc_maxima_sensor_id: int + _usage_forecast_sensor_id: int + _leakage_behaviour_sensor_id: int + _charging_efficiency_sensor_id: int _schedule_duration: timedelta - _valid_from_shift: timedelta + _fill_level_scale: float + _resolution = "15min" def __init__( self, @@ -40,34 +57,68 @@ def __init__( soc_sensor_id: int, rm_discharge_sensor_id: int, price_sensor_id: int, + production_price_sensor_id: int, + soc_minima_sensor_id: int, + soc_maxima_sensor_id: int, + usage_forecast_sensor_id: int, + leakage_behaviour_sensor_id: int, + charging_efficiency_sensor_id: int, timezone: str = "UTC", - schedule_duration: timedelta = timedelta(hours=12), + schedule_duration: timedelta = timedelta(hours=24), max_size: int = 100, - valid_from_shift: timedelta = timedelta(days=1), + fill_level_scale: float = 1, + power_unit: str = "W", + energy_unit: str = "J", ) -> None: super().__init__(max_size) self._power_sensor_id = power_sensor_id self._price_sensor_id = price_sensor_id + self._production_price_sensor_id = production_price_sensor_id self._schedule_duration = schedule_duration self._soc_sensor_id = soc_sensor_id self._rm_discharge_sensor_id = rm_discharge_sensor_id + self._soc_minima_sensor_id = soc_minima_sensor_id + self._soc_maxima_sensor_id = soc_maxima_sensor_id + self._usage_forecast_sensor_id = usage_forecast_sensor_id + self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id + self.charging_efficiency_sensor_id = charging_efficiency_sensor_id self._timezone = ZoneInfo(timezone) - - # delay the start of the schedule from the time `valid_from` - # of the FRBC.SystemDescription. - self._valid_from_shift = valid_from_shift + self._fill_level_scale = fill_level_scale + self.power_unit = power_unit + self.energy_unit = energy_unit def now(self): return datetime.now(self._timezone) + def _get_operation_mode_efficiency_sensor_map(self, system_description) -> dict: + """ + Map operation mode IDs to the charging efficiency sensor. + + For FRBCSimple, all operation modes report their efficiency to the + single charging_efficiency_sensor_id. + """ + efficiency_map = {} + if self.charging_efficiency_sensor_id is None: + return efficiency_map + + # Map each operation mode to the charging efficiency sensor + actuator = system_description.actuators[0] + for operation_mode in actuator.operation_modes: + efficiency_map[operation_mode.id] = self.charging_efficiency_sensor_id + + return efficiency_map + async def send_storage_status(self, status: FRBCStorageStatus): - await self._fm_client.post_measurements( + now = self.now() + await self._fm_client.post_sensor_data( self._soc_sensor_id, - start=self.now(), - values=[status.present_fill_level], - unit="MWh", + start=now, + prior=now, + values=[status.present_fill_level * self._fill_level_scale], + unit=self.energy_unit, duration=timedelta(minutes=1), ) + await self.trigger_schedule(now) async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor @@ -77,62 +128,226 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): power = ( fill_rate.start_of_range + (fill_rate.end_of_range - fill_rate.start_of_range) * factor - ) + ) * self._fill_level_scale - dt = status.transition_timestamp # self.now() + start = status.transition_timestamp or self.now() - await self._fm_client.post_measurements( - self._rm_discharge_sensor_id, - start=dt, - values=[-power], - unit="MWh", + await self._fm_client.post_sensor_data( + self._power_sensor_id, + start=start, + prior=self.now(), + values=[power], + unit=self.power_unit, duration=timedelta(minutes=15), ) - async def trigger_schedule(self, system_description_id: str): - """Translates S2 System Description into FM API calls""" + async def trigger_schedule( + self, start: datetime, system_description_id: str | None = None + ): + """ + Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager + """ - system_description: FRBCSystemDescription = self._system_description_history[ - system_description_id - ] + if system_description_id: + system_description: FRBCSystemDescription = ( + self._system_description_history[system_description_id] + ) + else: + # Use last SystemDescription + system_description: FRBCSystemDescription = list( + self._system_description_history.values() + )[-1] + system_descriptions = self._system_description_history.values() + self._logger.debug( + list( + [ + system_description.valid_from + for system_description in system_descriptions + ] + ) + ) + self._logger.debug(f"Using system description: {system_description}") if len(self._storage_status_history) > 0: - soc_at_start = list(self._storage_status_history.values())[ - -1 - ].present_fill_level + soc_at_start = ( + list(self._storage_status_history.values())[-1].present_fill_level + * self._fill_level_scale + ) else: print("Can't trigger schedule without knowing the status of the storage...") return - soc_min, soc_max = get_soc_min_max(system_description) + # Assume a single actuator + actuator = system_description.actuators[0] + + # Derive the overall power range + charging_capacity = None + discharging_capacity = None + for operation_mode in actuator.operation_modes: + for element in operation_mode.elements: + for power_range in element.power_ranges: + # todo: distinguish power range per commodity + p_min = power_range.end_of_range + if discharging_capacity is None: + discharging_capacity = p_min + else: + discharging_capacity = min(discharging_capacity, p_min) + p_max = power_range.start_of_range + if charging_capacity is None: + charging_capacity = p_max + else: + charging_capacity = max(charging_capacity, p_max) + + soc_min, soc_max = get_soc_min_max(system_description, self._fill_level_scale) + + # Support for J energy unit (FM server scheduling trigger endpoint only accepts kWh and MWh) + if self.energy_unit == "J": + f = 3.6 * 10**6 + energy_unit = "kWh" + soc_at_start /= f + soc_min /= f + soc_max /= f + else: + energy_unit = self.energy_unit # call schedule + if isinstance(start, str): + start = pd.Timestamp(start) schedule = await self._fm_client.trigger_and_get_schedule( - start=system_description.valid_from - + self._valid_from_shift, # TODO: localize datetime + start=start.replace( + minute=(start.minute // 15) * 15, second=0, microsecond=0 + ), + prior=start, sensor_id=self._power_sensor_id, flex_context={ - "production-price": {"sensor": self._price_sensor_id}, "consumption-price": {"sensor": self._price_sensor_id}, + "production-price": {"sensor": self._production_price_sensor_id}, "site-power-capacity": f"{3 * 25 * 230} VA", + "relax-soc-constraints": True, }, flex_model={ - "soc-unit": "MWh", - "soc-at-start": soc_at_start, # TODO: use forecast of the SOC instead + "soc-unit": energy_unit, + "soc-at-start": soc_at_start, "soc-min": soc_min, "soc-max": soc_max, + "soc-minima": {"sensor": self._soc_minima_sensor_id}, + "soc-maxima": {"sensor": self._soc_maxima_sensor_id}, + "state-of-charge": {"sensor": self._soc_sensor_id}, + "soc-usage": [{"sensor": self._usage_forecast_sensor_id}], + "storage-efficiency": {"sensor": self._leakage_behaviour_sensor_id}, + "charging-efficiency": {"sensor": self.charging_efficiency_sensor_id}, + "consumption-capacity": f"{charging_capacity} {self.power_unit}", + "production-capacity": f"{discharging_capacity} {self.power_unit}", }, duration=self._schedule_duration, # next 12 hours - prior=self.now(), # TODO: add SOC MAX AND SOC MIN FROM fill_level_range, # this needs changes on the client + unit=self.power_unit, ) # translate FlexMeasures schedule into instructions. SOC -> Power -> PowerFactor instructions = fm_schedule_to_instructions( - schedule, system_description, initial_fill_level=soc_at_start + schedule, + system_description, + initial_fill_level=soc_at_start / self._fill_level_scale, ) # put instructions to sending queue for instruction in instructions: - await self._sending_queue.put(instruction) + await self.send_message(instruction) + + async def send_fill_level_target_profile( + self, fill_level_target_profile: FRBCFillLevelTargetProfile + ): + """ + Send FRBC.FillLevelTargetProfile to FlexMeasures. + + Args: + fill_level_target_profile (FRBCFillLevelTargetProfile): The fill level target profile to be translated and sent. + """ + # if not self._is_timer_due("fill_level_target_profile"): + # return + + soc_minima, soc_maxima = translate_fill_level_target_profile( + fill_level_target_profile=fill_level_target_profile, + resolution=self._resolution, + fill_level_scale=self._fill_level_scale, + ) + + duration = str(pd.Timedelta(self._resolution) * len(soc_maxima)) + + # POST SOC Minima measurements to FlexMeasures + await self._fm_client.post_sensor_data( + sensor_id=self._soc_minima_sensor_id, + start=fill_level_target_profile.start_time, + prior=self.now(), + values=soc_minima.tolist(), + unit=self.energy_unit, + duration=duration, + ) + + # POST SOC Maxima measurements to FlexMeasures + await self._fm_client.post_sensor_data( + sensor_id=self._soc_maxima_sensor_id, + start=fill_level_target_profile.start_time, + prior=self.now(), + values=soc_maxima.tolist(), + unit=self.energy_unit, + duration=duration, + ) + + async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): + """ + Send FRBC.UsageForecast to FlexMeasures. + + Args: + usage_forecast (FRBCUsageForecast): The usage forecast to be translated and sent. + """ + # if not self._is_timer_due("usage_forecast"): + # return + + start_time = usage_forecast.start_time + + # flooring to previous 15min tick + start_time = start_time.replace( + minute=(start_time.minute // 15) * 15, second=0, microsecond=0 + ) + + usage_forecast = translate_usage_forecast_to_fm( + usage_forecast, + self._resolution, + strategy="mean", + fill_level_scale=self._fill_level_scale, + ) + + await self._fm_client.post_sensor_data( + sensor_id=self._usage_forecast_sensor_id, + start=start_time, + prior=self.now(), + values=usage_forecast.tolist(), + unit=self.power_unit, # e.g. [0, 100] MW/(15 min) # todo: or: f"{self.energy_unit}/s" to scale usage forecast e.g. [0, 100] %/s -> [0, 100] %/(15 min) + duration=str(pd.Timedelta(self._resolution) * len(usage_forecast)), + ) + + async def send_leakage_behaviour(self, leakage: FRBCLeakageBehaviour): + # if not self._is_timer_due("leakage_behaviour"): + # return + + start = leakage.valid_from or self.now() + start = start.replace(minute=(start.minute // 15) * 15, second=0, microsecond=0) + + storage_efficiency = leakage_behaviour_to_storage_efficiency( + message=leakage, + resolution=timedelta(minutes=15), + fill_level_scale=self._fill_level_scale, + ) + self._logger.debug(storage_efficiency) + + await self._fm_client.post_sensor_data( + self._leakage_behaviour_sensor_id, + start=start, + prior=self.now(), + values=[storage_efficiency], + unit="%", + duration=timedelta(hours=48), + ) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index e2496182..9f8224a8 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -123,7 +123,6 @@ def __init__( schedule_duration: timedelta = timedelta(hours=12), max_size: int = 100, fill_level_scale: float = 0.1, - valid_from_shift: timedelta = timedelta(days=1), timers: dict[str, datetime] | None = None, datastore: dict | None = None, **kwargs, @@ -154,10 +153,6 @@ def __init__( self._production_price_sensor_id = production_price_sensor self._timezone = ZoneInfo(timezone) - - # delay the start of the schedule from the time `valid_from` of the FRBC.SystemDescription - self._valid_from_shift = valid_from_shift - self._fill_level_scale = fill_level_scale self._active_recurring_schedule = False @@ -215,7 +210,7 @@ async def send_storage_status(self, status: FRBCStorageStatus): subject_message_id=status.message_id, status=ReceptionStatusValues.PERMANENT_ERROR, ) - await self._sending_queue.put(response) + await self.send_message(response) await self.trigger_schedule() async def send_leakage_behaviour(self, leakage: FRBCLeakageBehaviour): @@ -235,7 +230,7 @@ async def send_leakage_behaviour(self, leakage: FRBCLeakageBehaviour): leakage_behaviour_to_storage_efficiency( message=leakage, resolution=timedelta(minutes=15), - fill_level_scale=self._fill_level_scalefill_level_scale, + fill_level_scale=self._fill_level_scale, ) ], unit=PERCENTAGE, @@ -246,7 +241,7 @@ async def send_leakage_behaviour(self, leakage: FRBCLeakageBehaviour): subject_message_id=leakage.message_id, status=ReceptionStatusValues.PERMANENT_ERROR, ) - await self._sending_queue.put(response) + await self.send_message(response) async def send_actuator_status(self, status: FRBCActuatorStatus): if not self._is_timer_due("actuator_status"): @@ -514,12 +509,12 @@ async def trigger_schedule(self): object_id=message_id, ) self._logger.debug(f"Sending revoke instruction for {message_id}") - await self._sending_queue.put(revoke_instruction) + await self.send_message(revoke_instruction) self._datastore["instructions"] = {} # Put the instruction in the sending queue for instruction in instructions: - await self._sending_queue.put(instruction) + await self.send_message(instruction) # Store instructions for instruction in instructions: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 04ce1056..46cc7d4e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,4 +1,6 @@ +import json import logging +import uuid from datetime import timedelta from math import isclose from typing import List @@ -9,6 +11,7 @@ try: from s2python.common import NumberRange from s2python.frbc import ( + FRBCActuatorDescription, FRBCInstruction, FRBCLeakageBehaviour, FRBCOperationMode, @@ -155,12 +158,18 @@ def fm_schedule_to_instructions( f"{len(system_description.actuators)} were provided" ) - operation_modes: list[FRBCOperationMode] = actuator.operation_modes + actuators: dict[uuid.UUID, FRBCActuatorDescription] = { + a.id: a for a in system_description.actuators + } + operation_modes: dict[uuid.UUID, FRBCOperationMode] = { + om.id: om for om in actuator.operation_modes + } fill_level = initial_fill_level deltaT = timedelta(minutes=15) / timedelta(hours=1) + previous_instruction = None for timestamp, row in schedule.iterrows(): power = row["schedule"] usage = row.get("usage_forecast", 0) @@ -169,7 +178,7 @@ def fm_schedule_to_instructions( # Convert from power to fill rate results = [ (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) - for om in operation_modes + for om in operation_modes.values() ] # Step 1: minimize fill-level penalty (primary) @@ -221,10 +230,34 @@ def fm_schedule_to_instructions( execution_time=timestamp, abnormal_condition=False, ) + if previous_instruction and all( + getattr(previous_instruction, attr) == getattr(instruction, attr) + for attr in ( + "actuator_id", + "operation_mode", + "operation_mode_factor", + "abnormal_condition", + ) + ): + logger.info("Instruction removed, no changes to previous instruction") + continue logger.info( f"Instruction created: at {timestamp} set {actuator.diagnostic_label if isinstance(actuator.diagnostic_label, str) else actuator} to {best_operation_mode.diagnostic_label if isinstance(best_operation_mode.diagnostic_label, str) else best_operation_mode} with factor {operation_mode_factor}" ) + previous_instruction = instruction instructions.append(instruction) + logger.debug( + "Instructions JSON: %s", + json.dumps( + [ + serialize_instruction( + instr, actuators=actuators, operation_modes=operation_modes + ) + for instr in instructions + ], + indent=2, + ), + ) # Update fill level fill_level = compute_next_fill_level( @@ -381,3 +414,29 @@ def explain_choice( lines.append(f"{label} (element={element_label}): rejected due to {reason}") return "; ".join(lines) + + +def serialize_instruction( + instr: FRBCInstruction, + actuators: dict[uuid.UUID, FRBCActuatorDescription], + operation_modes: dict[uuid.UUID, FRBCOperationMode], +): + """Create dict of instructions suitable for logging.""" + actuator = ( + getattr(actuators[instr.actuator_id], "diagnostic_label", None) + or instr.actuator_id + ) + operation_mode = ( + getattr(operation_modes[instr.operation_mode], "diagnostic_label", None) + or instr.operation_mode + ) + return { + "message_type": instr.message_type, + "message_id": str(instr.message_id), + "instruction_id": str(instr.id), + "actuator": str(actuator), + "operation_mode": str(operation_mode), + "operation_mode_factor": instr.operation_mode_factor, + "execution_time": instr.execution_time.isoformat(), + "abnormal_condition": instr.abnormal_condition, + } diff --git a/src/flexmeasures_client/s2/control_types/__init__.py b/src/flexmeasures_client/s2/control_types/__init__.py index b3e19f5b..429ad419 100644 --- a/src/flexmeasures_client/s2/control_types/__init__.py +++ b/src/flexmeasures_client/s2/control_types/__init__.py @@ -1,8 +1,7 @@ from __future__ import annotations -from asyncio import Queue from logging import Logger -from typing import cast +from typing import Callable, cast from pydantic import BaseModel from s2python.common import ( @@ -22,7 +21,7 @@ class ControlTypeHandler(Handler): _instruction_history: SizeLimitOrderedDict[str, BaseModel] _instruction_status_history: SizeLimitOrderedDict[str, InstructionStatus] _fm_client: FlexMeasuresClient - _sending_queue: Queue + send_message: Callable _logger: Logger def __init__(self, max_size: int = 100) -> None: diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 6a11c948..7277a6d2 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -10,7 +10,6 @@ from flexmeasures_client.client import FlexMeasuresClient from flexmeasures_client.s2.cem import CEM -from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple log_level = os.getenv("LOGGING_LEVEL", "WARNING").upper() logging.basicConfig( @@ -29,31 +28,34 @@ async def rm_details_watchdog(ws, cem: CEM): """ # wait to get resource manager details - while cem._control_type is None: + while cem._control.control_type is None: await asyncio.sleep(1) await cem.activate_control_type(control_type=ControlType.FILL_RATE_BASED_CONTROL) # check/wait that the control type is set properly - while cem._control_type != ControlType.FILL_RATE_BASED_CONTROL: + while cem._control.control_type != ControlType.FILL_RATE_BASED_CONTROL: cem._logger.debug("waiting for the activation of the control type...") await asyncio.sleep(1) - cem._logger.debug(f"CONTROL TYPE: {cem._control_type}") + cem._logger.debug(f"CONTROL TYPE: {cem._control.control_type}") - # after this, schedule will be triggered on reception of a new system description + # after this, schedule will be triggered on reception of a new storage status async def websocket_producer(ws, cem: CEM): cem._logger.debug("start websocket message producer") cem._logger.debug(f"IS CLOSED? {cem.is_closed()}") while not cem.is_closed(): - message = await cem.get_message() - cem._logger.debug("sending message") + message, fut = await cem.get_message() try: + cem._logger.debug("sending message") await ws.send_json(message) - except aiohttp.ClientConnectionResetError: - break + fut.set_result(True) + # except aiohttp.ClientConnectionResetError: + # break + except Exception as exc: + fut.set_exception(exc) cem._logger.debug("cem closed") @@ -75,6 +77,7 @@ async def websocket_consumer(ws, cem: CEM): elif msg.type == aiohttp.WSMsgType.ERROR: cem._logger.debug("close...") await cem.close() + await ws.close() cem._logger.error(f"ws connection closed with exception {ws.exception()}") # TODO: save cem state? @@ -85,33 +88,23 @@ async def websocket_handler(request): ws = web.WebSocketResponse() await ws.prepare(request) - site_name = "My CEM" - base_url = os.getenv("FLEXMEASURES_BASE_URL", "http://localhost:5000") + base_url = os.getenv( + "FLEXMEASURES_BASE_URL", "http://localhost:5000" + ) # or "server:5000" parsed = urlparse(base_url) fm_client = FlexMeasuresClient( password=os.getenv("FLEXMEASURES_PASSWORD", "toy-password"), email=os.getenv("FLEXMEASURES_USER", "toy-user@flexmeasures.io"), host=parsed.netloc, ssl=parsed.scheme == "https", - ) - - price_sensor, power_sensor, soc_sensor, rm_discharge_sensor = await configure_site( - site_name, fm_client + polling_interval=0.5, ) cem = CEM( - sensor_id=power_sensor["id"], + power_sensor_id=None, # assign CEM a top-level asset directly fm_client=fm_client, logger=LOGGER, ) - frbc = FRBCSimple( - power_sensor_id=power_sensor["id"], - price_sensor_id=price_sensor["id"], - soc_sensor_id=soc_sensor["id"], - rm_discharge_sensor_id=rm_discharge_sensor["id"], - ) - cem.register_control_type(frbc) - # create "parallel" tasks for the message producer and consumer await asyncio.gather( websocket_consumer(ws, cem), @@ -122,116 +115,6 @@ async def websocket_handler(request): return ws -async def configure_site( - site_name: str, fm_client: FlexMeasuresClient -) -> tuple[dict, dict, dict, dict]: - account = await fm_client.get_account() - assets = await fm_client.get_assets(parse_json_fields=True) - - site_asset = None - for asset in assets: - if asset["name"] == site_name: - site_asset = asset - break - - site_asset_specs = dict( - latitude=0, - longitude=0, - generic_asset_type_id=6, # Building asset type - flex_model={ - "power-capacity": f"{3 * 25 * 230} VA", - }, - ) - - if not site_asset: - site_asset = await fm_client.add_asset( - name=site_name, account_id=account["id"], **site_asset_specs - ) - # Update site asset with the latest specs - await fm_client.update_asset(site_asset["id"], site_asset_specs) - - sensors = site_asset.get("sensors", []) - price_sensor = None - power_sensor = None - soc_sensor = None - rm_discharge_sensor = None - for sensor in sensors: - if sensor["name"] == "price": - price_sensor = sensor - elif sensor["name"] == "power": - power_sensor = sensor - elif sensor["name"] == "state of charge": - soc_sensor = sensor - elif sensor["name"] == "RM discharge": - rm_discharge_sensor = sensor - - if price_sensor is None: - price_sensor = await fm_client.add_sensor( - name="price", - event_resolution="PT15M", - unit="EUR/kWh", - generic_asset_id=site_asset["id"], - timezone="Europe/Amsterdam", - ) - await fm_client.post_sensor_data( - sensor_id=price_sensor["id"], - start="2026-01-15T00:00+01", # 2026-01-01T00:00+01 - duration="P3D", # P1M - values=[ - 0.10, - 0.11, - 0.12, - 0.15, - 0.18, - 0.17, - 0.11, - 0.09, - 0.10, - 0.09, - 0.09, - 0.10, - 0.08, - 0.05, - 0.04, - 0.04, - 0.06, - 0.08, - 0.12, - 0.13, - 0.14, - 0.13, - 0.10, - 0.07, - ], - unit="EUR/kWh", - ) - if power_sensor is None: - power_sensor = await fm_client.add_sensor( - name="power", - event_resolution="PT15M", - unit="kW", - generic_asset_id=site_asset["id"], - timezone="Europe/Amsterdam", - ) - if soc_sensor is None: - soc_sensor = await fm_client.add_sensor( - name="state of charge", - event_resolution="PT0M", - unit="kWh", - generic_asset_id=site_asset["id"], - timezone="Europe/Amsterdam", - ) - if rm_discharge_sensor is None: - rm_discharge_sensor = await fm_client.add_sensor( - name="RM discharge", - event_resolution="PT15M", - unit="dimensionless", - generic_asset_id=site_asset["id"], - timezone="Europe/Amsterdam", - ) - return price_sensor, power_sensor, soc_sensor, rm_discharge_sensor - - app = web.Application() app.add_routes([web.get("/ws", websocket_handler)]) web.run_app(app) diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index 50eabaab..b7a2f502 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import OrderedDict +from dataclasses import dataclass, field from typing import Mapping, TypeVar from uuid import uuid4 @@ -9,7 +10,7 @@ from packaging.version import Version try: - from s2python.common import ReceptionStatus, ReceptionStatusValues + from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues except ImportError: raise ImportError( "The 's2-python' package is required for this functionality. " @@ -39,6 +40,12 @@ def __setitem__(self, __key: KT, __value: VT) -> None: return super().__setitem__(__key, __value) +@dataclass +class ControlContext: + control_type: ControlType | None = None + handler_ready: dict[ControlType, bool] = field(default_factory=dict) + + def get_unique_id() -> str: """Generate a random v4 UUID string. diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 37f3aaff..2b997efe 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -1,12 +1,16 @@ from __future__ import annotations +import asyncio import pytest from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues +from unittest.mock import AsyncMock, MagicMock from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC import FRBCTest + + @pytest.mark.asyncio async def test_handshake(rm_handshake): cem = CEM(fm_client=None) @@ -26,8 +30,8 @@ async def test_handshake(rm_handshake): ) # check that two messages are put to the outgoing queue (ReceptionStatus and HandshakeResponse) # CEM response - response = await cem.get_message() # ReceptionStatus for Handshake - response = await cem.get_message() # HandshakeResponse + response, _ = await cem.get_message() # ReceptionStatus for Handshake + response, _ = await cem.get_message() # HandshakeResponse assert ( response["message_type"] == "HandshakeResponse" @@ -54,8 +58,8 @@ async def test_resource_manager_details(resource_manager_details, rm_handshake): cem._sending_queue.qsize() == 2 ) # check that message is put to the outgoing queue - response = await cem.get_message() # ReceptionStatus for Handshake - response = await cem.get_message() # HandshakeResponse + response, _ = await cem.get_message() # ReceptionStatus for Handshake + response, _ = await cem.get_message() # HandshakeResponse ########################## # ResourceManagerDetails # @@ -63,7 +67,7 @@ async def test_resource_manager_details(resource_manager_details, rm_handshake): # RM sends ResourceManagerDetails await cem.handle_message(resource_manager_details) - response = await cem.get_message() + response, _ = await cem.get_message() # CEM response is ReceptionStatus with an OK status assert response["message_type"] == "ReceptionStatus" @@ -77,6 +81,15 @@ async def test_resource_manager_details(resource_manager_details, rm_handshake): "independently of the original type" ) + # Cleanup: cancel any pending background tasks + for task_id, task in cem._handler_build_tasks.items(): + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + @pytest.mark.asyncio async def test_activate_control_type( @@ -92,14 +105,14 @@ async def test_activate_control_type( ############# await cem.handle_message(rm_handshake) - response = await cem.get_message() # ReceptionStatus for Handshake - response = await cem.get_message() # HandshakeResponse + response, _ = await cem.get_message() # ReceptionStatus for Handshake + response, _ = await cem.get_message() # HandshakeResponse ########################## # ResourceManagerDetails # ########################## await cem.handle_message(resource_manager_details) - response = await cem.get_message() + response, _ = await cem.get_message() ######################### # Activate control type # @@ -107,7 +120,7 @@ async def test_activate_control_type( # CEM sends a request to change te control type await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) - message = await cem.get_message() + message, _ = await cem.get_message() assert cem.control_type == ControlType.NO_SELECTION, ( "the control type should still be NO_SELECTION (rather than FRBC)," @@ -124,6 +137,15 @@ async def test_activate_control_type( cem.control_type == ControlType.FILL_RATE_BASED_CONTROL ), "after a positive ResponseStatus, the status changes from NO_SELECTION to FRBC" + # Cleanup: cancel any pending background tasks + for task_id, task in cem._handler_build_tasks.items(): + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + @pytest.mark.asyncio async def test_messages_route_to_control_type_handler( @@ -139,21 +161,21 @@ async def test_messages_route_to_control_type_handler( ############# await cem.handle_message(rm_handshake) - response = await cem.get_message() # ReceptionStatus for Handshake - response = await cem.get_message() # HandshakeResponse + response, _ = await cem.get_message() # ReceptionStatus for Handshake + response, _ = await cem.get_message() # HandshakeResponse ########################## # ResourceManagerDetails # ########################## await cem.handle_message(resource_manager_details) - response = await cem.get_message() + response, _ = await cem.get_message() ######################### # Activate control type # ######################### await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) - message = await cem.get_message() + message, _ = await cem.get_message() response = ReceptionStatus( subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK @@ -166,7 +188,7 @@ async def test_messages_route_to_control_type_handler( ######## await cem.handle_message(frbc_system_description) - response = await cem.get_message() + response, _ = await cem.get_message() # checking that FRBC handler is being called assert ( @@ -182,9 +204,9 @@ async def test_messages_route_to_control_type_handler( # change of control type is not performed in case that the RM answers # with a negative response await cem.activate_control_type(ControlType.NO_SELECTION) - response = await cem.get_message() + response, _ = await cem.get_message() assert ( - cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + cem._control.control_type == ControlType.FILL_RATE_BASED_CONTROL ), "control type should not change, confirmation still pending" await cem.handle_message( @@ -195,7 +217,7 @@ async def test_messages_route_to_control_type_handler( ) assert ( - cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + cem._control.control_type == ControlType.FILL_RATE_BASED_CONTROL ), "control type should not change, confirmation state is not 'OK'" assert ( response.get("message_id") @@ -204,6 +226,15 @@ async def test_messages_route_to_control_type_handler( ].success_callbacks ), "success callback should be deleted" + # Cleanup: cancel any pending background tasks + for task_id, task in cem._handler_build_tasks.items(): + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + @pytest.mark.asyncio async def test_automatic_change_control_type(resource_manager_details, rm_handshake): @@ -222,8 +253,8 @@ async def test_automatic_change_control_type(resource_manager_details, rm_handsh cem._sending_queue.qsize() == 2 ) # check that message is put to the outgoing queue - response = await cem.get_message() - response = await cem.get_message() # HandshakeResponse + response, _ = await cem.get_message() + response, _ = await cem.get_message() # HandshakeResponse ########################## # ResourceManagerDetails # @@ -231,14 +262,81 @@ async def test_automatic_change_control_type(resource_manager_details, rm_handsh # RM sends ResourceManagerDetails await cem.handle_message(resource_manager_details) - response = await cem.get_message() + response, _ = await cem.get_message() # CEM sends control type on receiving the ResourceManagerDetails assert response["message_type"] == "SelectControlType" assert response["control_type"] == "FILL_RATE_BASED_CONTROL" - response = await cem.get_message() + response, _ = await cem.get_message() # CEM response is ReceptionStatus with an OK status assert response["message_type"] == "ReceptionStatus" assert response["status"] == "OK" + + # Cleanup: cancel any pending background tasks + for task_id, task in cem._handler_build_tasks.items(): + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_handle_message_during_handler_registration_race(): + cem = CEM( + fm_client=MagicMock(), + logger=MagicMock(), + ) + + # --- Fake FRBC handler that we will "register late" + frbc_handler = AsyncMock() + frbc_handler._control_type = ControlType.FILL_RATE_BASED_CONTROL + frbc_handler.supports_message.return_value = True + frbc_handler.handle_message.return_value = {"ok": True} + + # Simulate slow map_resource_to_asset + registration_started = asyncio.Event() + registration_continue = asyncio.Event() + + async def slow_map_resource_to_asset(message): + registration_started.set() + await registration_continue.wait() + + cem.register_control_type(frbc_handler) + + cem.map_resource_to_asset = slow_map_resource_to_asset + + # Set control type BEFORE handler exists + cem.update_control_type(ControlType.FILL_RATE_BASED_CONTROL) + + # Start async registration + task = asyncio.create_task( + cem.map_resource_to_asset(MagicMock(resource_id="x", name="test")) + ) + + # Wait until registration has started but not finished + await registration_started.wait() + + # --- THIS is the race moment + msg = {"message_type": "TestMessage", "message_id": "550e8400-e29b-41d4-a716-446655440000"} + + # Should NOT crash even though handler isn't registered yet + await cem.handle_message(msg) + + # A response should be queued (ReceptionStatus with TEMPORARY_ERROR since handler not ready) + response, _ = await cem.get_message() + assert response.get("message_type") == "ReceptionStatus" + assert response.get("status") == "TEMPORARY_ERROR" + + # Now finish registration + registration_continue.set() + await task + + # Now handler should have been called the second time + await cem.handle_message(msg) + + assert frbc_handler.handle_message.called + diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 645692c9..0efb5832 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -78,21 +78,21 @@ async def setup_cem(resource_manager_details, rm_handshake): ############# await cem.handle_message(rm_handshake) - response = await cem.get_message() # ReceptionStatus for Handshake - response = await cem.get_message() # HandshakeResponse + response, _ = await cem.get_message() # ReceptionStatus for Handshake + response, _ = await cem.get_message() # HandshakeResponse ########################## # ResourceManagerDetails # ########################## await cem.handle_message(resource_manager_details) - response = await cem.get_message() + response, _ = await cem.get_message() ######################### # Activate control type # ######################### await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) - message = await cem.get_message() + message, _ = await cem.get_message() response = ReceptionStatus( subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK @@ -112,7 +112,7 @@ async def cem_in_frbc_control_type(setup_cem, frbc_system_description): ######## await cem.handle_message(frbc_system_description) - await cem.get_message() + _, _ = await cem.get_message() return cem, fm_client, frbc_system_description diff --git a/tests/s2/test_timezone.py b/tests/s2/test_timezone.py index 59ab27b6..11039fcf 100644 --- a/tests/s2/test_timezone.py +++ b/tests/s2/test_timezone.py @@ -16,6 +16,12 @@ def test_frbc_simple_now_uses_zoneinfo_timezone(): soc_sensor_id=2, rm_discharge_sensor_id=3, price_sensor_id=4, + production_price_sensor_id=5, + soc_minima_sensor_id=6, + soc_maxima_sensor_id=7, + usage_forecast_sensor_id=8, + leakage_behaviour_sensor_id=9, + charging_efficiency_sensor_id=10, timezone="Europe/Amsterdam", )