diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 8ab2780d6b..e28b45f618 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -42,6 +42,7 @@ Infrastructure / Support * Speed up sensor stats, especially potent when viewing sensors stats over a large sensor history [`PR #2173 `_] * Various smaller fixes in documenting scheduling endpoints and flex-model fields [`PR #2122 `_] * Add cross-cutting Copilot instruction files to ``.github/instructions/`` covering atomic commits, changelog format, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, timezone awareness, and UI terminology [see `PR #2198 `_] +* Speed up per-function test isolation by introducing a ``transactional_db`` fixture that wraps each test in a rolled-back transaction instead of recreating the DB schema; applied to ``test_sensor_schedules_fresh_db.py`` [see `PR #2204 `_] Bugfixes ----------- diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 200243b1fe..8588154d3c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -40,17 +40,17 @@ ) def test_trigger_and_get_schedule( app, - add_market_prices_fresh_db, - add_battery_assets_fresh_db, - battery_soc_sensor_fresh_db, - add_charging_station_assets_fresh_db, + add_market_prices_transactional_db, + add_battery_assets_transactional_db, + battery_soc_sensor_transactional_db, + add_charging_station_assets_transactional_db, keep_scheduling_queue_empty, message, asset_name, requesting_user, ): # noqa: C901 # Include the price sensor and site-power-capacity in the flex-context explicitly, to test deserialization - price_sensor_id = add_market_prices_fresh_db["epex_da"].id + price_sensor_id = add_market_prices_transactional_db["epex_da"].id message["flex-context"] = { "consumption-price": {"sensor": price_sensor_id}, "production-price": {"sensor": price_sensor_id}, @@ -207,35 +207,39 @@ def test_trigger_and_get_schedule( ) def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( app, - fresh_db, - add_market_prices_fresh_db, - add_battery_assets_fresh_db, - battery_soc_sensor_fresh_db, - setup_sources_fresh_db, + transactional_db, + add_market_prices_transactional_db, + add_battery_assets_transactional_db, + battery_soc_sensor_transactional_db, + setup_sources_transactional_db, keep_scheduling_queue_empty, requesting_user, ): message = message_for_trigger_schedule(resolution="PT1H") message["flex-context"] = { - "consumption-price": {"sensor": add_market_prices_fresh_db["epex_da"].id}, - "production-price": {"sensor": add_market_prices_fresh_db["epex_da"].id}, + "consumption-price": { + "sensor": add_market_prices_transactional_db["epex_da"].id + }, + "production-price": { + "sensor": add_market_prices_transactional_db["epex_da"].id + }, "site-power-capacity": "1 TW", } message["flex-model"]["state-of-charge"] = { - "sensor": battery_soc_sensor_fresh_db.id + "sensor": battery_soc_sensor_transactional_db.id } message["flex-model"].pop("soc-at-start") - fresh_db.session.add( + transactional_db.session.add( TimedBelief( - sensor=battery_soc_sensor_fresh_db, - source=setup_sources_fresh_db["Seita"], + sensor=battery_soc_sensor_transactional_db, + source=setup_sources_transactional_db["Seita"], event_start=parse_datetime(message["start"]), belief_horizon=timedelta(0), event_value=50, ) ) - fresh_db.session.commit() + transactional_db.session.commit() sensor = ( Sensor.query.filter(Sensor.name == "power") @@ -257,7 +261,9 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( with app.test_client() as client: get_soc_schedule_response = client.get( url_for( - "SensorAPI:get_schedule", id=battery_soc_sensor_fresh_db.id, uuid=job_id + "SensorAPI:get_schedule", + id=battery_soc_sensor_transactional_db.id, + uuid=job_id, ), query_string={"duration": "PT24H"}, ) @@ -265,7 +271,7 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( assert get_soc_schedule_response.json["unit"] == "%" assert get_soc_schedule_response.json["values"][0] == 50 - sensor = fresh_db.session.get(Sensor, sensor.id) + sensor = transactional_db.session.get(Sensor, sensor.id) assert sensor.generic_asset.get_attribute("soc_in_mwh") == pytest.approx(0.02) @@ -296,11 +302,11 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( ) def test_price_sensor_priority( app, - fresh_db, - add_market_prices_fresh_db, - add_battery_assets_fresh_db, - battery_soc_sensor_fresh_db, - add_charging_station_assets_fresh_db, + transactional_db, + add_market_prices_transactional_db, + add_battery_assets_transactional_db, + battery_soc_sensor_transactional_db, + add_charging_station_assets_transactional_db, keep_scheduling_queue_empty, context_sensor, asset_sensor, @@ -325,23 +331,23 @@ def test_price_sensor_priority( sensor_attribute = f"{sensor_type}-price" # preparation: ensure the asset actually has the price sensor set in its flex_context if asset_sensor: - price_sensor_id = add_market_prices_fresh_db[asset_sensor].id - battery_asset = add_battery_assets_fresh_db[asset_name] + price_sensor_id = add_market_prices_transactional_db[asset_sensor].id + battery_asset = add_battery_assets_transactional_db[asset_name] battery_asset.flex_context[sensor_attribute] = {"sensor": price_sensor_id} - fresh_db.session.add(battery_asset) + transactional_db.session.add(battery_asset) if parent_sensor: - price_sensor_id = add_market_prices_fresh_db[parent_sensor].id - building_asset = add_battery_assets_fresh_db["Test building"] + price_sensor_id = add_market_prices_transactional_db[parent_sensor].id + building_asset = add_battery_assets_transactional_db["Test building"] building_asset.flex_context[sensor_attribute] = {"sensor": price_sensor_id} - fresh_db.session.add(building_asset) + transactional_db.session.add(building_asset) # Adding unused sensor to context (e.g. consumption price sensor if we test production sensor) message["flex-context"] = { - unused_sensor: {"sensor": add_market_prices_fresh_db["epex_da"].id}, + unused_sensor: {"sensor": add_market_prices_transactional_db["epex_da"].id}, "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities } if context_sensor: - price_sensor_id = add_market_prices_fresh_db[context_sensor].id + price_sensor_id = add_market_prices_transactional_db[context_sensor].id message["flex-context"][used_sensor] = {"sensor": price_sensor_id} # trigger a schedule through the /sensors//schedules/trigger [POST] api endpoint @@ -367,7 +373,7 @@ def test_price_sensor_priority( ) as patched_search_beliefs: work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) - expect_price_sensor_id = add_market_prices_fresh_db[expect_sensor].id + expect_price_sensor_id = add_market_prices_transactional_db[expect_sensor].id # TimedBelief.search is called twice for a price sensor: 1st call has consumption price sensor, 2nd call has production price sensor call_num = 0 if sensor_type == "consumption" else 1 call_args = patched_search_beliefs.call_args_list[call_num] @@ -393,11 +399,11 @@ def test_price_sensor_priority( ) def test_inflexible_device_sensors_priority( app, - fresh_db, - add_market_prices_fresh_db, - add_battery_assets_fresh_db, - battery_soc_sensor_fresh_db, - add_charging_station_assets_fresh_db, + transactional_db, + add_market_prices_transactional_db, + add_battery_assets_transactional_db, + battery_soc_sensor_transactional_db, + add_charging_station_assets_transactional_db, keep_scheduling_queue_empty, context_sensor_num, asset_sensor_num, @@ -408,32 +414,35 @@ def test_inflexible_device_sensors_priority( message, asset_name = message_for_trigger_schedule(), "Test battery" message["force_new_job_creation"] = True - price_sensor_id = add_market_prices_fresh_db["epex_da"].id + price_sensor_id = add_market_prices_transactional_db["epex_da"].id message["flex-context"] = { "consumption-price": {"sensor": price_sensor_id}, "production-price": {"sensor": price_sensor_id}, "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities } if context_sensor_num: - other_asset = add_battery_assets_fresh_db["Test small battery"] + other_asset = add_battery_assets_transactional_db["Test small battery"] context_sensors = setup_inflexible_device_sensors( - fresh_db, other_asset, "other asset sensors", context_sensor_num + transactional_db, other_asset, "other asset sensors", context_sensor_num ) message["flex-context"]["inflexible-device-sensors"] = [ sensor.id for sensor in context_sensors ] if asset_sensor_num: - battery_asset = add_battery_assets_fresh_db[asset_name] + battery_asset = add_battery_assets_transactional_db[asset_name] battery_sensors = setup_inflexible_device_sensors( - fresh_db, battery_asset, "battery asset sensors", asset_sensor_num + transactional_db, battery_asset, "battery asset sensors", asset_sensor_num ) - link_sensors(fresh_db, battery_asset, battery_sensors) + link_sensors(transactional_db, battery_asset, battery_sensors) if parent_sensor_num: - building_asset = add_battery_assets_fresh_db["Test building"] + building_asset = add_battery_assets_transactional_db["Test building"] building_sensors = setup_inflexible_device_sensors( - fresh_db, building_asset, "building asset sensors", parent_sensor_num + transactional_db, + building_asset, + "building asset sensors", + parent_sensor_num, ) - link_sensors(fresh_db, building_asset, building_sensors) + link_sensors(transactional_db, building_asset, building_sensors) # trigger a schedule through the /sensors//schedules/trigger [POST] api endpoint assert len(app.queues["scheduling"]) == 0 @@ -475,10 +484,10 @@ def test_inflexible_device_sensors_priority( ) def test_multiple_contracts( app, - fresh_db, - add_market_prices_fresh_db, - add_battery_assets_fresh_db, - battery_soc_sensor_fresh_db, + transactional_db, + add_market_prices_transactional_db, + add_battery_assets_transactional_db, + battery_soc_sensor_transactional_db, include_consumption_breach, requesting_user, ): @@ -504,7 +513,7 @@ def test_multiple_contracts( # we won't need to breach this site_consumption_capacity to reach the target site_consumption_capacity = 1 - price_sensor_id = add_market_prices_fresh_db["epex_da"].id + price_sensor_id = add_market_prices_transactional_db["epex_da"].id message["flex-context"] = { "consumption-price": {"sensor": price_sensor_id}, "production-price": {"sensor": price_sensor_id}, @@ -610,7 +619,7 @@ def test_multiple_contracts( ) -def setup_inflexible_device_sensors(fresh_db, asset, sensor_name, sensor_num): +def setup_inflexible_device_sensors(db, asset, sensor_name, sensor_num): """Test helper function to add sensor_num sensors to an asset""" sensors = list() for i in range(sensor_num): @@ -621,16 +630,16 @@ def setup_inflexible_device_sensors(fresh_db, asset, sensor_name, sensor_num): unit="MW", attributes={"capacity_in_mw": 2}, ) - fresh_db.session.add(sensor) + db.session.add(sensor) sensors.append(sensor) - fresh_db.session.flush() + db.session.flush() return sensors -def link_sensors(fresh_db, asset, sensors): +def link_sensors(db, asset, sensors): asset.flex_context.setdefault("inflexible-device-sensors", list()) asset.flex_context["inflexible-device-sensors"].extend( [sensor.id for sensor in sensors] ) - fresh_db.session.add(asset) + db.session.add(asset) diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 06767abeee..0011c44a8c 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -89,6 +89,40 @@ def fresh_db(app): yield test_db +@pytest.fixture(scope="function") +def transactional_db(db): + """Function-scoped DB fixture that uses transaction rollback for test isolation. + + Faster than fresh_db because it avoids recreating the DB schema for each test. + Instead, each test is wrapped in a transaction (with savepoints for commits) + that is rolled back at teardown, leaving the DB in its pre-test state. + + All DB operations within a test — including setup fixtures, HTTP requests through + the Flask test client, and in-process RQ workers — share the same session and + therefore the same transaction, so the rollback undoes all of them. + + Requires a module-scoped ``db`` fixture to ensure the schema already exists. + Tests that need a truly empty database (e.g. DDL-manipulating migration tests) + should continue to use ``fresh_db`` instead. + """ + from sqlalchemy.orm import Session as SASession + + connection = db.engine.connect() + transaction = connection.begin() + # join_transaction_mode="create_savepoint" means session.commit() creates a + # SAVEPOINT and releases it rather than committing the outer transaction, so + # the outer transaction.rollback() in teardown can undo everything. + session = SASession(bind=connection, join_transaction_mode="create_savepoint") + db.session.registry.set(session) + + yield db + + session.close() + db.session.registry.clear() + transaction.rollback() + connection.close() + + @contextmanager def create_test_db(app: Flask): """ @@ -123,6 +157,11 @@ def setup_accounts_fresh_db(fresh_db) -> dict[str, Account]: return create_test_accounts(fresh_db) +@pytest.fixture(scope="function") +def setup_accounts_transactional_db(transactional_db) -> dict[str, Account]: + return create_test_accounts(transactional_db) + + def create_test_accounts(db) -> dict[str, Account]: prosumer_account_role = AccountRole(name="Prosumer", description="A Prosumer") prosumer_account = Account( @@ -199,6 +238,13 @@ def setup_roles_users_fresh_db(fresh_db, setup_accounts_fresh_db) -> dict[str, U return create_roles_users(fresh_db, setup_accounts_fresh_db) +@pytest.fixture(scope="function") +def setup_roles_users_transactional_db( + transactional_db, setup_accounts_transactional_db +) -> dict[str, User]: + return create_roles_users(transactional_db, setup_accounts_transactional_db) + + def create_roles_users(db, test_accounts) -> dict[str, User]: """Create a minimal set of roles and users""" new_users: list[User] = [] @@ -325,6 +371,11 @@ def setup_markets_fresh_db(fresh_db) -> dict[str, Sensor]: return create_test_markets(fresh_db) +@pytest.fixture(scope="function") +def setup_markets_transactional_db(transactional_db) -> dict[str, Sensor]: + return create_test_markets(transactional_db) + + def create_test_markets(db) -> dict[str, Sensor]: """Create the epex_da market.""" @@ -368,6 +419,11 @@ def setup_sources_fresh_db(fresh_db) -> dict[str, DataSource]: return create_sources(fresh_db) +@pytest.fixture(scope="function") +def setup_sources_transactional_db(transactional_db) -> dict[str, DataSource]: + return create_sources(transactional_db) + + def create_sources(db) -> dict[str, DataSource]: seita_source = DataSource(name="Seita", type="demo script") db.session.add(seita_source) @@ -406,6 +462,20 @@ def setup_generic_assets_fresh_db( ) +@pytest.fixture(scope="function") +def setup_generic_assets_transactional_db( + transactional_db, + setup_generic_asset_types_transactional_db, + setup_accounts_transactional_db, +) -> dict[str, GenericAsset]: + """Make some generic assets used throughout.""" + return create_generic_assets( + transactional_db, + setup_generic_asset_types_transactional_db, + setup_accounts_transactional_db, + ) + + def create_generic_assets( db, setup_generic_asset_types, setup_accounts ) -> dict[str, GenericAsset]: @@ -453,6 +523,14 @@ def setup_generic_asset_types_fresh_db(fresh_db) -> dict[str, GenericAssetType]: return create_generic_asset_types(fresh_db) +@pytest.fixture(scope="function") +def setup_generic_asset_types_transactional_db( + transactional_db, +) -> dict[str, GenericAssetType]: + """Make some generic asset types used throughout.""" + return create_generic_asset_types(transactional_db) + + def create_generic_asset_types(db) -> dict[str, GenericAssetType]: public_good = GenericAssetType( name="public good", @@ -507,6 +585,23 @@ def setup_assets_fresh_db( ) +@pytest.fixture(scope="function") +def setup_assets_transactional_db( + transactional_db, + setup_accounts_transactional_db, + setup_markets_transactional_db, + setup_sources_transactional_db, + setup_generic_asset_types_transactional_db, +) -> dict[str, GenericAsset]: + return create_assets( + transactional_db, + setup_accounts_transactional_db, + setup_markets_transactional_db, + setup_sources_transactional_db, + setup_generic_asset_types_transactional_db, + ) + + def create_assets( db, setup_accounts, setup_markets, setup_sources, setup_asset_types ) -> dict[str, GenericAsset]: @@ -709,6 +804,21 @@ def add_market_prices_fresh_db( ) +@pytest.fixture(scope="function") +def add_market_prices_transactional_db( + transactional_db: SQLAlchemy, + setup_assets_transactional_db, + setup_markets_transactional_db, + setup_sources_transactional_db, +) -> dict[str, Sensor]: + return add_market_prices_common( + transactional_db, + setup_assets_transactional_db, + setup_markets_transactional_db, + setup_sources_transactional_db, + ) + + def add_market_prices_common( db: SQLAlchemy, setup_assets, setup_markets, setup_sources ) -> dict[str, Sensor]: @@ -861,6 +971,22 @@ def add_battery_assets_fresh_db( ) +@pytest.fixture(scope="function") +def add_battery_assets_transactional_db( + transactional_db, + setup_roles_users_transactional_db, + setup_accounts_transactional_db, + setup_markets_transactional_db, + setup_generic_asset_types_transactional_db, +) -> dict[str, GenericAsset]: + return create_test_battery_assets( + transactional_db, + setup_accounts_transactional_db, + setup_markets_transactional_db, + setup_generic_asset_types_transactional_db, + ) + + def create_test_battery_kWh_assets( db: SQLAlchemy, setup_accounts, setup_markets, generic_asset_types ) -> dict[str, GenericAsset]: @@ -1249,6 +1375,19 @@ def add_charging_station_assets_fresh_db( ) +@pytest.fixture(scope="function") +def add_charging_station_assets_transactional_db( + transactional_db: SQLAlchemy, + setup_accounts_transactional_db, + setup_markets_transactional_db, +) -> dict[str, GenericAsset]: + return create_charging_station_assets( + transactional_db, + setup_accounts_transactional_db, + setup_markets_transactional_db, + ) + + def create_charging_station_assets( db: SQLAlchemy, setup_accounts, setup_markets ) -> dict[str, GenericAsset]: @@ -1482,6 +1621,16 @@ def battery_soc_sensor_fresh_db(fresh_db: SQLAlchemy, setup_generic_assets_fresh return create_battery_soc_sensor(fresh_db, setup_generic_assets_fresh_db) +@pytest.fixture(scope="function") +def battery_soc_sensor_transactional_db( + transactional_db: SQLAlchemy, setup_generic_assets_transactional_db +): + """Add a battery SOC sensor to the transactional db.""" + return create_battery_soc_sensor( + transactional_db, setup_generic_assets_transactional_db + ) + + def create_battery_soc_sensor(db: SQLAlchemy, setup_generic_assets): """Add a battery SOC sensor.""" soc_sensor = Sensor( diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 2bd1321271..a0269a9be1 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -1162,7 +1162,7 @@ def test_numerical_errors(app_with_each_solver, setup_planning_test_data, db): tz = pytz.timezone("Europe/Amsterdam") start = tz.localize(datetime(2015, 1, 2)) - end = tz.localize(datetime(2015, 1, 3)) + end = start + timedelta(hours=4) resolution = timedelta(minutes=5) duration_until_next_target = timedelta(hours=1)