From dcc3719a7a8d97c838fc776f7ba14ff9241c3a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20A=2E=20Rodr=C3=ADguez?= Date: Tue, 12 May 2026 14:36:46 +0200 Subject: [PATCH 1/2] Add system test for KPI integration in simulator workflow - Add test__simulator__kpis_present_in_output asserting KPIs are present in instance[0].area after a full pipeline run, with CAPEX verified against the ESDL costInformation and system_lifetime exercised with a non-default value (25 years) - Update docker-compose.yml to use simulator-worker 0.0.30beta2 (kpi-calculator 0.4.2 integration; 0.0.29 was never released as stable) - Add EXCLUDE_KPIS path set and pass it to compare_esdl in existing simulator tests so KPI churn does not break snapshot comparisons - Increase simulator test timeouts to 120s to account for KPI calculation time added after simulation --- docker-compose.yml | 2 +- system_tests/src/test_workflows_steps.py | 103 +++++++++++++++++++++-- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4c70b3d..97fb47d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -215,7 +215,7 @@ services: - "./gurobi/gurobi.lic:/app/gurobi/gurobi.lic" omotes_simulator_worker: - image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.28 + image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.30beta2 restart: unless-stopped deploy: replicas: 2 diff --git a/system_tests/src/test_workflows_steps.py b/system_tests/src/test_workflows_steps.py index 8edbe4f..902c14c 100644 --- a/system_tests/src/test_workflows_steps.py +++ b/system_tests/src/test_workflows_steps.py @@ -37,6 +37,12 @@ port=int(os.environ.get("RABBITMQ_PORT", "5672")), ) +# KPI content is tested separately in test__simulator__kpis_present_in_output +# and changes with each kpi-calculator release, so exclude it from snapshot comparisons. +# Note: this path assumes xmltodict parses a single element (dict, not list). +# If multi-instance ESDLs are introduced, this path will need to be updated. +EXCLUDE_KPIS: set[str] = {"root['esdl:EnergySystem']['instance']['area']['KPIs']"} + SQL_CONFIG = { "host": os.environ.get("POSTGRES_HOST", "localhost"), "port": int(os.environ.get("POSTGRES_PORT", "6432")), @@ -225,10 +231,19 @@ def expect_a_result( f"The job did not finish as {expected_result}. Found {result.result_type}" ) - def compare_esdl(self, expected_esdl: str, result_esdl: str) -> None: + def compare_esdl( + self, + expected_esdl: str, + result_esdl: str, + exclude_paths: set[str] | None = None, + ) -> None: + """Compare two ESDL strings for equality after normalization. + + :param exclude_paths: Optional DeepDiff paths to ignore (e.g. EXCLUDE_KPIS). + """ expected = normalize_esdl(expected_esdl) result = normalize_esdl(result_esdl) - diff = DeepDiff(expected, result, math_epsilon=ESDL_VALUES_PRECISION) + diff = DeepDiff(expected, result, math_epsilon=ESDL_VALUES_PRECISION, exclude_paths=exclude_paths) if diff: diff_msg = pformat(diff) @@ -283,7 +298,7 @@ def test__simulator__happy_path(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), @@ -302,7 +317,9 @@ def test__simulator__happy_path(self) -> None: expected_esdl = retrieve_esdl_file( "./test_esdl/output/test__simulator__happy_path.esdl" ) - self.compare_esdl(expected_esdl, result_handler.result.output_esdl) + self.compare_esdl( + expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPIS + ) # assert time series data created assert_influxdb_database_existence(result_handler.result.output_esdl, True) @@ -322,7 +339,7 @@ def test__simulator__multiple_ates_run(self) -> None: "./test_esdl/input/simulator_ates_short_run.esdl" ) workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), @@ -350,7 +367,9 @@ def test__simulator__multiple_ates_run(self) -> None: for result_handler in result_handlers: self.expect_a_result(result_handler, JobResult.SUCCEEDED) - self.compare_esdl(expected_esdl, result_handler.result.output_esdl) + self.compare_esdl( + expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPIS + ) def test__grow_optimizer_default__happy_path_1source(self) -> None: # Arrange @@ -431,7 +450,7 @@ def test__simulator__job_reference_is_set(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0), @@ -459,7 +478,7 @@ def test__simulator__check_if_progress_updates_are_received(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 60.0 + timeout_seconds = 120.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0), @@ -612,7 +631,7 @@ def test__simulator__delete_time_series_data_after_run(self) -> None: result_handler = OmotesJobHandler() esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") workflow_type = "simulator" - timeout_seconds = 100.0 + timeout_seconds = 160.0 params_dict = { "timestep": datetime.timedelta(hours=1), "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), @@ -722,3 +741,69 @@ def _watch_job(result_handler: OmotesJobHandler): self.assertTrue(str(high_priority_job.id) in ordered_job_result_ids) # check that high priority job result was not last (exact order may vary) self.assertNotEqual(str(high_priority_job.id), ordered_job_result_ids[-1]) + + def test__simulator__kpis_present_in_output(self) -> None: + """Test that KPIs are calculated and stored in the output ESDL. + + Uses simulator_ates_short_run.esdl which contains costInformation on assets, + allowing the kpi-calculator to produce non-trivial KPI results. + """ + # Arrange + result_handler = OmotesJobHandler() + esdl_file = retrieve_esdl_file( + "./test_esdl/input/simulator_ates_short_run.esdl" + ) + workflow_type = "simulator" + timeout_seconds = 120.0 + params_dict = { + "timestep": datetime.timedelta(hours=1), + "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), + "end_time": datetime.datetime(2019, 1, 1, 3, 0, 0, tzinfo=datetime.UTC), + "system_lifetime": 25.0, + } + + # Act + with omotes_client() as omotes_client_: + submit_a_job( + omotes_client_, esdl_file, workflow_type, params_dict, result_handler + ) + result_handler.wait_until_result(timeout_seconds) + + # Assert + self.expect_a_result(result_handler, JobResult.SUCCEEDED) + output_esh = esdl.esdl_handler.EnergySystemHandler() + output_esh.load_from_string(result_handler.result.output_esdl) + energy_system = output_esh.energy_system + + # KPIs are attached to instance[0].area, not energy_system directly + self.assertGreater( + len(energy_system.instance), 0, "Output ESDL must have at least one instance" + ) + main_area = energy_system.instance[0].area + self.assertIsNotNone(main_area, "instance[0] must have an area") + self.assertIsNotNone(main_area.KPIs, "KPIs should be present in the main area") + kpi_list = list(main_area.KPIs.kpi) + self.assertGreater(len(kpi_list), 0, "At least one KPI should be calculated") + for kpi in kpi_list: + self.assertNotEqual(kpi.name, "", "Each KPI should have a name") + + # CAPEX: the ATES asset in simulator_ates_short_run.esdl has + # investmentCosts=2333594.0 EUR (bare EUR unit, no conversion factor). + # This is the only asset with cost data so total CAPEX equals that value. + expected_capex = 2_333_594.0 + kpi_by_name = {kpi.name: kpi for kpi in kpi_list} + self.assertIn( + "High level cost breakdown [EUR]", + kpi_by_name, + "Cost breakdown KPI missing from output", + ) + cost_items = { + item.label: item.value + for item in kpi_by_name["High level cost breakdown [EUR]"].distribution.stringItem + } + self.assertAlmostEqual( + cost_items.get("CAPEX (total)", 0.0), + expected_capex, + places=1, + msg=f"CAPEX should match investmentCosts in simulator_ates_short_run.esdl; got {cost_items}", + ) From 19d0df1e325143eb3895039564f3503dd710e538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20A=2E=20Rodr=C3=ADguez?= Date: Tue, 12 May 2026 15:17:19 +0200 Subject: [PATCH 2/2] Fix ATES simulator tests for omotes-simulator-core 0.0.30 - Replace simulator_ates_short_run.esdl with test_ates.esdl from simulator-worker to fix negative mass flow caused by reversed ATES pipe directions - Update snapshot test__simulator__multiple_ates_run.esdl to match new ESDL output - Improve KPI assertions in test__simulator__kpis_present_in_output: check exact KPI name set, assert CAPEX/OPEX values against ESDL cost data, and verify Production/Demand energy entries are positive - Update docker-compose.yml to use simulator-worker 0.0.30 --- docker-compose.yml | 2 +- .../input/simulator_ates_short_run.esdl | 264 ++-- .../test__simulator__multiple_ates_run.esdl | 1230 +++++++++-------- system_tests/src/test_workflows_steps.py | 50 +- 4 files changed, 803 insertions(+), 743 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 97fb47d..183d77e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -215,7 +215,7 @@ services: - "./gurobi/gurobi.lic:/app/gurobi/gurobi.lic" omotes_simulator_worker: - image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.30beta2 + image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.30 restart: unless-stopped deploy: replicas: 2 diff --git a/system_tests/src/test_esdl/input/simulator_ates_short_run.esdl b/system_tests/src/test_esdl/input/simulator_ates_short_run.esdl index ea0eac1..bb39034 100644 --- a/system_tests/src/test_esdl/input/simulator_ates_short_run.esdl +++ b/system_tests/src/test_esdl/input/simulator_ates_short_run.esdl @@ -1,73 +1,73 @@ - + - - - - - - - + + + + + + + - - + + - - + + - + - + - - + + - + - + - - + + - + - - - - + + + + - + - - + + - + - - + + - - - - + + + + - + @@ -75,10 +75,10 @@ - - + + - + @@ -89,71 +89,71 @@ - - + + - - - - + + + + - - - - + + + + - + - - - + + + - - + + - + - - + + - - + + - + - - - - - + + + + + - - + + - + - - - - - - - - + + + + + + + + - - + + - + - - + + - + @@ -161,84 +161,84 @@ - - + + - - - - + + + + - + - - - + + + - - + + - + - - + + - - + + - + - - + + - - - + + + - - + + - + - + - - + + - + + + + + + + + + + + + - + - + - - - - + - - - - - - - - - - - - + + + + - \ No newline at end of file + diff --git a/system_tests/src/test_esdl/output/test__simulator__multiple_ates_run.esdl b/system_tests/src/test_esdl/output/test__simulator__multiple_ates_run.esdl index 2409ade..aec94fb 100644 --- a/system_tests/src/test_esdl/output/test__simulator__multiple_ates_run.esdl +++ b/system_tests/src/test_esdl/output/test__simulator__multiple_ates_run.esdl @@ -1,307 +1,307 @@ - + - - - - - - - + + - - + + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - + @@ -309,64 +309,64 @@ - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + @@ -377,395 +377,395 @@ - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - - - + + + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - - - - - - + + + + + + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + @@ -773,429 +773,463 @@ - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - + + + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + - - - - - + + + + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/system_tests/src/test_workflows_steps.py b/system_tests/src/test_workflows_steps.py index 902c14c..935b296 100644 --- a/system_tests/src/test_workflows_steps.py +++ b/system_tests/src/test_workflows_steps.py @@ -41,7 +41,7 @@ # and changes with each kpi-calculator release, so exclude it from snapshot comparisons. # Note: this path assumes xmltodict parses a single element (dict, not list). # If multi-instance ESDLs are introduced, this path will need to be updated. -EXCLUDE_KPIS: set[str] = {"root['esdl:EnergySystem']['instance']['area']['KPIs']"} +EXCLUDE_KPI_PATHS: set[str] = {"root['esdl:EnergySystem']['instance']['area']['KPIs']"} SQL_CONFIG = { "host": os.environ.get("POSTGRES_HOST", "localhost"), @@ -239,7 +239,7 @@ def compare_esdl( ) -> None: """Compare two ESDL strings for equality after normalization. - :param exclude_paths: Optional DeepDiff paths to ignore (e.g. EXCLUDE_KPIS). + :param exclude_paths: Optional DeepDiff paths to ignore (e.g. EXCLUDE_KPI_PATHS). """ expected = normalize_esdl(expected_esdl) result = normalize_esdl(result_esdl) @@ -318,7 +318,7 @@ def test__simulator__happy_path(self) -> None: "./test_esdl/output/test__simulator__happy_path.esdl" ) self.compare_esdl( - expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPIS + expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPI_PATHS ) # assert time series data created @@ -368,7 +368,7 @@ def test__simulator__multiple_ates_run(self) -> None: for result_handler in result_handlers: self.expect_a_result(result_handler, JobResult.SUCCEEDED) self.compare_esdl( - expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPIS + expected_esdl, result_handler.result.output_esdl, exclude_paths=EXCLUDE_KPI_PATHS ) def test__grow_optimizer_default__happy_path_1source(self) -> None: @@ -787,23 +787,49 @@ def test__simulator__kpis_present_in_output(self) -> None: for kpi in kpi_list: self.assertNotEqual(kpi.name, "", "Each KPI should have a name") - # CAPEX: the ATES asset in simulator_ates_short_run.esdl has - # investmentCosts=2333594.0 EUR (bare EUR unit, no conversion factor). - # This is the only asset with cost data so total CAPEX equals that value. - expected_capex = 2_333_594.0 kpi_by_name = {kpi.name: kpi for kpi in kpi_list} - self.assertIn( + + # Assert all expected KPI categories are present + expected_kpi_names = { "High level cost breakdown [EUR]", - kpi_by_name, - "Cost breakdown KPI missing from output", + "Net Present Value [EUR]", + "Equivalent Annual Cost [EUR/yr]", + "Total Cost of Ownership [EUR]", + "Energy breakdown [Wh]", + } + self.assertEqual( + expected_kpi_names, + set(kpi_by_name.keys()), + "KPI names do not match expected set", ) + + # CAPEX: the ATES asset in simulator_ates_short_run.esdl has + # investmentCosts=2333594.0 EUR — only asset with cost data so total CAPEX equals that. + # OPEX: fixedOperationalCosts(30000) + fixedMaintenanceCosts(115472.22) + # + variableOperationalCosts(69666.67) = 215138.89 EUR/yr — asset-level, time-independent. cost_items = { item.label: item.value for item in kpi_by_name["High level cost breakdown [EUR]"].distribution.stringItem } self.assertAlmostEqual( cost_items.get("CAPEX (total)", 0.0), - expected_capex, + 2_333_594.0, places=1, msg=f"CAPEX should match investmentCosts in simulator_ates_short_run.esdl; got {cost_items}", ) + self.assertAlmostEqual( + cost_items.get("OPEX (yearly)", 0.0), + 215_138.89, + places=1, + msg=f"OPEX should match sum of fixed/variable costs in simulator_ates_short_run.esdl; got {cost_items}", + ) + + # Energy breakdown: confirms time-series results flowed through the simulation + energy_items = { + item.label: item.value + for item in kpi_by_name["Energy breakdown [Wh]"].distribution.stringItem + } + self.assertIn("Production", energy_items, "Energy breakdown should contain Production") + self.assertIn("Demand", energy_items, "Energy breakdown should contain Demand") + self.assertGreater(energy_items["Production"], 0.0, "Production energy should be positive") + self.assertGreater(energy_items["Demand"], 0.0, "Demand energy should be positive")