From a42b648bd49d758c8dfabdfd501859aedccfb067 Mon Sep 17 00:00:00 2001 From: Devansh Sharma <46666171+devansh287@users.noreply.github.com> Date: Fri, 8 Aug 2025 12:30:55 +0530 Subject: [PATCH 1/7] Updated sensor specs and save_forecasts I added a separate sensor name for each API service. The save_forecasts_to_db function now checks which service we are using, and accordingly maps the value we are interested in to the relevant Flexmeasures sensor. Signed-off-by: Devansh Sharma <46666171+devansh287@users.noreply.github.com> --- flexmeasures_weather/sensor_specs.py | 12 ++++-- flexmeasures_weather/utils/weather.py | 56 ++++++--------------------- 2 files changed, 20 insertions(+), 48 deletions(-) diff --git a/flexmeasures_weather/sensor_specs.py b/flexmeasures_weather/sensor_specs.py index 1e06cc6..6a1dcaa 100644 --- a/flexmeasures_weather/sensor_specs.py +++ b/flexmeasures_weather/sensor_specs.py @@ -18,28 +18,32 @@ mapping = [ dict( fm_sensor_name="temperature", - weather_sensor_name="temp", + OWM_sensor_name="temp", + WAPI_sensor_name="temp_c", unit="°C", event_resolution=timedelta(minutes=60), attributes=weather_attributes, ), dict( fm_sensor_name="wind speed", - weather_sensor_name="wind_speed", + OWM_sensor_name="wind_speed", + WAPI_sensor_name="wind_kph", unit="m/s", event_resolution=timedelta(minutes=60), attributes=weather_attributes, ), dict( fm_sensor_name="cloud cover", - weather_sensor_name="clouds", + OWM_sensor_name="clouds", + WAPI_sensor_name="cloud", unit="%", event_resolution=timedelta(minutes=60), attributes=weather_attributes, ), dict( fm_sensor_name="irradiance", # in save_forecasts_to_db, we catch this name and do the actual computation to get to the irradiance - weather_sensor_name="clouds", + OWM_sensor_name="clouds", + WAPI_sensor_name="cloud", unit="W/m²", event_resolution=timedelta(minutes=60), attributes=weather_attributes, diff --git a/flexmeasures_weather/utils/weather.py b/flexmeasures_weather/utils/weather.py index f27f91a..e7c725f 100644 --- a/flexmeasures_weather/utils/weather.py +++ b/flexmeasures_weather/utils/weather.py @@ -68,45 +68,7 @@ def process_weatherapi_data( combined = first_day + second_day + third_day relevant = combined[hour_no : hour_no + 48] - # relevant = combined - - def map_weather_api_to_owm(weather_api_data: Dict[str, Any]) -> Dict[str, Any]: - """ - Converts a single hour of WeatherAPI data to an OpenWeatherMap-style dictionary. - - Args: - weather_api_data (Dict[str, Any]): A dictionary containing an hour's data from WeatherAPI. - - Returns: - Dict[str, Any]: A dictionary with keys and structure similar to OpenWeatherMap's hourly forecast. - """ - game = { - "dt": weather_api_data["time_epoch"], - "temp": weather_api_data["temp_c"], - "feels_like": weather_api_data["feelslike_c"], - "pressure": weather_api_data["pressure_mb"], - "humidity": weather_api_data["humidity"], - "dew_point": weather_api_data["dewpoint_c"], - "uvi": weather_api_data["uv"], - "clouds": weather_api_data["cloud"], - "visibility": weather_api_data["vis_km"] * 1000, - "wind_speed": weather_api_data["wind_kph"] / 3.6, - "wind_deg": weather_api_data["wind_degree"], - "wind_gust": weather_api_data["gust_kph"] / 3.6, - "weather": [ - { - "id": weather_api_data["condition"]["code"], - "main": weather_api_data["condition"]["text"].split()[0], - "description": weather_api_data["condition"]["text"], - "icon": weather_api_data["condition"]["icon"], - } - ], - "pop": weather_api_data["chance_of_rain"] / 100, - } - return game - - converted = [map_weather_api_to_owm(hour) for hour in relevant] - return converted + return relevant def call_openweatherapi( @@ -226,6 +188,7 @@ def save_forecasts_in_db( "WEATHER_MAXIMAL_DEGREE_LOCATION_DISTANCE", DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE, ) + provider = str(current_app.config.get("WEATHER_PROVIDER", "")) for location in locations: click.echo("[FLEXMEASURES] %s, %s" % location) weather_sensors: Dict[str, Sensor] = ( @@ -246,8 +209,9 @@ def save_forecasts_in_db( # loop through forecasts, including the one of current hour (horizon 0) for fc in forecasts: + time_key = fc["dt"] if provider == "OWM" else fc["time_epoch"] fc_datetime = as_server_time( - datetime.fromtimestamp(fc["dt"], get_timezone()) + datetime.fromtimestamp(time_key, get_timezone()) ) click.echo( f"[FLEXMEASURES-WEATHER] Processing forecast for {fc_datetime} ..." @@ -255,8 +219,8 @@ def save_forecasts_in_db( data_source = get_or_create_owm_data_source() for sensor_specs in mapping: sensor_name = str(sensor_specs["fm_sensor_name"]) - owm_response_label = sensor_specs["weather_sensor_name"] - if owm_response_label in fc: + provider_response_label = sensor_specs[f"{provider}_sensor_name"] + if provider_response_label in fc: weather_sensor = get_weather_sensor( sensor_specs, location, @@ -270,7 +234,11 @@ def save_forecasts_in_db( if weather_sensor not in db_forecasts.keys(): db_forecasts[weather_sensor] = [] - fc_value = fc[owm_response_label] + fc_value = fc[provider_response_label] + + if provider_response_label == 'wind_kph': + # convert wind speed from kph to m/s + fc_value = fc[provider_response_label] / 3.6 # the irradiance is not available in Provider -> we compute it ourselves if sensor_name == "irradiance": @@ -297,7 +265,7 @@ def save_forecasts_in_db( else: # we will not fail here, but issue a warning msg = "No label '%s' in response data for time %s" % ( - owm_response_label, + provider_response_label, fc_datetime, ) click.echo("[FLEXMEASURES-WEATHER] %s" % msg) From 672ef8ad66eba2760af741e884326860015536d3 Mon Sep 17 00:00:00 2001 From: Devansh Sharma <46666171+devansh287@users.noreply.github.com> Date: Fri, 8 Aug 2025 12:32:25 +0530 Subject: [PATCH 2/7] Making test response as per provider Here we ensure that the mock API response is as per the API service provider. Also popping the values for the API service provider sensors in commands. Signed-off-by: Devansh Sharma <46666171+devansh287@users.noreply.github.com> --- flexmeasures_weather/cli/commands.py | 3 ++- flexmeasures_weather/cli/tests/utils.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/flexmeasures_weather/cli/commands.py b/flexmeasures_weather/cli/commands.py index a95c435..00cc148 100644 --- a/flexmeasures_weather/cli/commands.py +++ b/flexmeasures_weather/cli/commands.py @@ -98,7 +98,8 @@ def add_weather_sensor(**args): fm_sensor_specs["generic_asset"] = weather_station fm_sensor_specs["timezone"] = args["timezone"] fm_sensor_specs["name"] = fm_sensor_specs.pop("fm_sensor_name") - fm_sensor_specs.pop("weather_sensor_name") + fm_sensor_specs.pop("OWM_sensor_name") + fm_sensor_specs.pop("WAPI_sensor_name") sensor = Sensor(**fm_sensor_specs) sensor.attributes = fm_sensor_specs["attributes"] diff --git a/flexmeasures_weather/cli/tests/utils.py b/flexmeasures_weather/cli/tests/utils.py index 3dab3ab..9c87578 100644 --- a/flexmeasures_weather/cli/tests/utils.py +++ b/flexmeasures_weather/cli/tests/utils.py @@ -1,6 +1,6 @@ from typing import List from datetime import datetime, timedelta - +from flask import current_app from flexmeasures.utils.time_utils import as_server_time, get_timezone @@ -17,11 +17,21 @@ def mock_api_response(api_key, location): mock_date_tz_aware = as_server_time( datetime.fromtimestamp(mock_date.timestamp(), tz=get_timezone()) ).replace(second=0, microsecond=0) + + provider = str(current_app.config.get("WEATHER_PROVIDER", "")) + date_key = "dt" + temp_key = "temp" + wind_speed_key = "wind_speed" + if provider == "WAPI": + date_key = "time_epoch" + temp_key = "temp" + wind_speed_key = "wind_kph" + return mock_date_tz_aware, [ - {"dt": mock_date.timestamp(), "temp": 40, "wind_speed": 100}, + {date_key: mock_date.timestamp(), temp_key: 40, wind_speed_key: 100}, { - "dt": (mock_date + timedelta(hours=1)).timestamp(), - "temp": 42, - "wind_speed": 90, + date_key: (mock_date + timedelta(hours=1)).timestamp(), + temp_key: 42, + wind_speed_key: 90, }, ] From db6543172958269775cd23798605310080347aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 31 Oct 2025 13:22:04 +0100 Subject: [PATCH 3/7] run black formatter to fix a few lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures_weather/utils/weather.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures_weather/utils/weather.py b/flexmeasures_weather/utils/weather.py index e7c725f..7d500dd 100644 --- a/flexmeasures_weather/utils/weather.py +++ b/flexmeasures_weather/utils/weather.py @@ -174,7 +174,7 @@ def call_api( return call_weatherapi(api_key, location) -def save_forecasts_in_db( +def save_forecasts_in_db( # noqa: C901 api_key: str, locations: List[Tuple[float, float]], ): @@ -236,9 +236,9 @@ def save_forecasts_in_db( fc_value = fc[provider_response_label] - if provider_response_label == 'wind_kph': + if provider_response_label == "wind_kph": # convert wind speed from kph to m/s - fc_value = fc[provider_response_label] / 3.6 + fc_value = fc[provider_response_label] / 3.6 # the irradiance is not available in Provider -> we compute it ourselves if sensor_name == "irradiance": From fa81a917df9d21407772c3d7168edeaaac7e1a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 28 May 2026 12:20:15 +0200 Subject: [PATCH 4/7] fix tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- README.md | 16 ++++++++- .../cli/tests/test_get_forecasts.py | 35 +++++++++++++++++++ flexmeasures_weather/cli/tests/utils.py | 2 +- flexmeasures_weather/utils/weather.py | 6 +++- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 92fc269..22a2522 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,21 @@ To expand the plugin's coverage to additional weather API services: This function should return data in the same structure as used by the original OpenWeatherMap integration, and **must have at least 48 hours of forecast data from the time of the call**. -3. **Integrate into the plugin** + You also need a provider-specific mapping entry in `flexmeasures_weather/sensor_specs.py`. Each supported sensor should include the new provider's response field name, for example: + + ```python + dict( + fm_sensor_name="temperature", + OWM_sensor_name="temp", + WAPI_sensor_name="temp_c", + NEWAPI_sensor_name="temperatureC", + unit="°C", + event_resolution=timedelta(minutes=60), + attributes=weather_attributes, + ) + ``` + +3. **Integrate into the plugin** Modify the `call_api` function in the `weather.py` file to include a conditional branch for the new provider: ```python diff --git a/flexmeasures_weather/cli/tests/test_get_forecasts.py b/flexmeasures_weather/cli/tests/test_get_forecasts.py index 2f33497..c26a1e5 100644 --- a/flexmeasures_weather/cli/tests/test_get_forecasts.py +++ b/flexmeasures_weather/cli/tests/test_get_forecasts.py @@ -1,5 +1,6 @@ import logging +import pytest from flexmeasures.data.models.time_series import TimedBelief from ..commands import collect_weather_data @@ -24,6 +25,7 @@ def test_get_weather_forecasts_to_db( weather_station = wind_sensor.generic_asset monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy") + monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM") monkeypatch.setattr(weather, "call_api", mock_api_response) runner = app.test_cli_runner() @@ -44,6 +46,38 @@ def test_get_weather_forecasts_to_db( assert wind_speed in [belief.event_value for belief in beliefs] +def test_get_weather_forecasts_wapi_mapping( + app, fresh_db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db +): + """ + Test that WeatherAPI provider-specific field names are mapped independently. + """ + wind_sensor = add_weather_sensors_fresh_db["wind"] + fresh_db.session.flush() + wind_sensor_id = wind_sensor.id + weather_station = wind_sensor.generic_asset + + monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy") + monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "WAPI") + monkeypatch.setattr(weather, "call_api", mock_api_response) + + runner = app.test_cli_runner() + result = runner.invoke( + collect_weather_data, + ["--location", f"{weather_station.latitude},{weather_station.longitude}"], + ) + assert "Reported task get-weather-forecasts status as True" in result.output + + beliefs = ( + fresh_db.session.query(TimedBelief) + .filter(TimedBelief.sensor_id == wind_sensor_id) + .all() + ) + assert len(beliefs) == 2 + expected_values = [pytest.approx(100 / 3.6), pytest.approx(90 / 3.6)] + assert [belief.event_value for belief in beliefs] == expected_values + + def test_get_weather_forecasts_no_close_sensors( app, db, monkeypatch, run_as_cli, add_weather_sensors_fresh_db, caplog ): @@ -54,6 +88,7 @@ def test_get_weather_forecasts_no_close_sensors( weather_station = add_weather_sensors_fresh_db["wind"].generic_asset monkeypatch.setitem(app.config, "WEATHERAPI_KEY", "dummy") + monkeypatch.setitem(app.config, "WEATHER_PROVIDER", "OWM") monkeypatch.setattr(weather, "call_api", mock_api_response) runner = app.test_cli_runner() diff --git a/flexmeasures_weather/cli/tests/utils.py b/flexmeasures_weather/cli/tests/utils.py index 9c87578..fdb001c 100644 --- a/flexmeasures_weather/cli/tests/utils.py +++ b/flexmeasures_weather/cli/tests/utils.py @@ -24,7 +24,7 @@ def mock_api_response(api_key, location): wind_speed_key = "wind_speed" if provider == "WAPI": date_key = "time_epoch" - temp_key = "temp" + temp_key = "temp_c" wind_speed_key = "wind_kph" return mock_date_tz_aware, [ diff --git a/flexmeasures_weather/utils/weather.py b/flexmeasures_weather/utils/weather.py index 7d500dd..9021ef6 100644 --- a/flexmeasures_weather/utils/weather.py +++ b/flexmeasures_weather/utils/weather.py @@ -189,6 +189,10 @@ def save_forecasts_in_db( # noqa: C901 DEFAULT_MAXIMAL_DEGREE_LOCATION_DISTANCE, ) provider = str(current_app.config.get("WEATHER_PROVIDER", "")) + if provider not in ["OWM", "WAPI"]: + raise Exception( + "Invalid provider name. Please set WEATHER_PROVIDER setting in config file to either OWM or WAPI, the two permissible options." + ) for location in locations: click.echo("[FLEXMEASURES] %s, %s" % location) weather_sensors: Dict[str, Sensor] = ( @@ -204,7 +208,7 @@ def save_forecasts_in_db( # noqa: C901 f"[FLEXMEASURES-WEATHER] Warning: difference between this server and Weather Provider is {naturaldelta(diff_fm_owm)}" ) click.echo( - f"[FLEXMEASURES-WEATHER] Called OpenWeatherMap API successfully at {now}." + f"[FLEXMEASURES-WEATHER] Called weather provider {provider} API successfully at {now}." ) # loop through forecasts, including the one of current hour (horizon 0) From 90c7808d1509f4e6c35e05b825631eddb85a3243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 5 Jun 2026 12:59:54 +0200 Subject: [PATCH 5/7] add quotes to example settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 22a2522..630e6b4 100644 --- a/README.md +++ b/README.md @@ -61,17 +61,17 @@ Add the following entries to your config: ```ini # Select the weather provider to use: "OWM" (OpenWeatherMap) or "WAPI" (Weather API) -WEATHER_PROVIDER = OWM +WEATHER_PROVIDER = "OWM" # API key for the selected weather provider -WEATHERAPI_KEY = your-api-key-here +WEATHERAPI_KEY = "your-api-key-here" # Name to register the weather data source in FlexMeasures. The default is 'Weather'. # Examples: "OpenWeatherMap" (for backwards compatibility with the OWM plugin). -WEATHER_DATA_SOURCE_NAME = 'OpenWeatherMap' +WEATHER_DATA_SOURCE_NAME = "OpenWeatherMap" # File path to store weather data in JSON format -WEATHER_FILE_PATH_LOCATION = /path/to/weather_output.json +WEATHER_FILE_PATH_LOCATION = "/path/to/weather_output.json" ``` ### Extending to Other Weather API Services From 1457e52b2cde59e377d22494d31c66baaefa39b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 5 Jun 2026 13:00:41 +0200 Subject: [PATCH 6/7] remove marshmallow limit which has gone from FM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- requirements/app.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/app.in b/requirements/app.in index 86168a4..b26161a 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -1,6 +1,4 @@ flexmeasures -# remove when FlexMeasures also removes this -marshmallow>=3,<4 pvlib # the following three are optional in pvlib, but we use them netCDF4 From eaff941752785cacd3a536b519ab8d6f2aa6c702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 5 Jun 2026 13:01:13 +0200 Subject: [PATCH 7/7] fix wrong query for case when --asset-id is passed in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures_weather/utils/locating.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures_weather/utils/locating.py b/flexmeasures_weather/utils/locating.py index 555872c..9a21b1c 100644 --- a/flexmeasures_weather/utils/locating.py +++ b/flexmeasures_weather/utils/locating.py @@ -114,9 +114,7 @@ def find_weather_sensor_by_location( def get_location_by_asset_id(asset_id: int) -> Tuple[float, float]: """Get location for forecasting by passing an asset id""" - asset = GenericAsset.query.filter( - GenericAsset.generic_asset_type_id == asset_id - ).one_or_none() + asset = GenericAsset.query.filter(GenericAsset.id == asset_id).one_or_none() if asset.generic_asset_type.name != WEATHER_STATION_TYPE_NAME: raise Exception( f"Asset {asset} does not seem to be a weather station we should use ― we expect an asset with type '{WEATHER_STATION_TYPE_NAME}'."