From 7114f208232ce003d2b122f053eca7cd88351afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Wed, 29 Apr 2026 14:10:40 +0200 Subject: [PATCH 01/11] Fix error message that mentions the wrong run model --- src/ert/run_models/ensemble_smoother.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ert/run_models/ensemble_smoother.py b/src/ert/run_models/ensemble_smoother.py index 8a34fc40430..1e4d13bfed3 100644 --- a/src/ert/run_models/ensemble_smoother.py +++ b/src/ert/run_models/ensemble_smoother.py @@ -37,7 +37,7 @@ def run_experiment( ) -> None: self.log_at_startup() if rerun_failed_realizations: - raise ErtRunError("Ensemble Information Filter does not support restart") + raise ErtRunError("Ensemble Smoother does not support restart") self.run_workflows(fixtures=PreExperimentFixtures(random_seed=self.random_seed)) From 5bd3802ffb983f6c597477a763a8676bcbfecbab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Wed, 29 Apr 2026 22:12:37 +0200 Subject: [PATCH 02/11] Add discriminator type to run model configs --- src/ert/run_models/ensemble_experiment.py | 3 ++- src/ert/run_models/ensemble_information_filter.py | 6 ++++-- src/ert/run_models/ensemble_smoother.py | 4 +++- src/ert/run_models/evaluate_ensemble.py | 3 ++- src/ert/run_models/manual_update.py | 3 ++- src/ert/run_models/manual_update_enif.py | 9 +++++++-- src/ert/run_models/multiple_data_assimilation.py | 3 ++- src/ert/run_models/single_test_run.py | 5 ++++- src/everest/config/everest_config.py | 4 +++- .../heat_equationconfig.ert/config.json | 1 + .../poly_examplepoly.ert/poly.json | 1 + .../snake_oilsnake_oil.ert/snake_oil.json | 1 + .../heat_equationconfig.ert/config.json | 1 + .../poly_examplepoly.ert/poly.json | 1 + .../snake_oilsnake_oil.ert/snake_oil.json | 1 + .../heat_equationconfig.ert/config.json | 1 + .../poly_examplepoly.ert/poly.json | 1 + .../snake_oilsnake_oil.ert/snake_oil.json | 1 + .../heat_equationconfig.ert/config.json | 1 + .../poly_examplepoly.ert/poly.json | 1 + .../snake_oilsnake_oil.ert/snake_oil.json | 1 + .../heat_equationconfig.ert/config.json | 1 + .../poly_examplepoly.ert/poly.json | 1 + .../snake_oilsnake_oil.ert/snake_oil.json | 1 + .../heat_equationconfig.ert/config.json | 1 + .../poly_examplepoly.ert/poly.json | 1 + .../snake_oilsnake_oil.ert/snake_oil.json | 1 + 27 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/ert/run_models/ensemble_experiment.py b/src/ert/run_models/ensemble_experiment.py index 6a4754973e0..945579c1e8a 100644 --- a/src/ert/run_models/ensemble_experiment.py +++ b/src/ert/run_models/ensemble_experiment.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import ClassVar +from typing import ClassVar, Literal from uuid import UUID from pydantic import PrivateAttr @@ -23,6 +23,7 @@ class EnsembleExperimentConfig(InitialEnsembleRunModelConfig): + type: Literal["ensemble_experiment"] = "ensemble_experiment" target_ensemble: str supports_rerunning_failed_realizations: ClassVar[bool] = True diff --git a/src/ert/run_models/ensemble_information_filter.py b/src/ert/run_models/ensemble_information_filter.py index 8303e8e80ad..2ffb454456d 100644 --- a/src/ert/run_models/ensemble_information_filter.py +++ b/src/ert/run_models/ensemble_information_filter.py @@ -2,6 +2,7 @@ import functools import logging +from typing import Literal from ert.analysis import enif_update from ert.run_models.ensemble_smoother import EnsembleSmoother @@ -16,10 +17,11 @@ class EnsembleInformationFilterConfig( InitialEnsembleRunModelConfig, UpdateRunModelConfig -): ... +): + type: Literal["ensemble_information_filter"] = "ensemble_information_filter" -class EnsembleInformationFilter(EnsembleSmoother, EnsembleInformationFilterConfig): +class EnsembleInformationFilter(EnsembleInformationFilterConfig, EnsembleSmoother): def update_ensemble_parameters( self, prior: Ensemble, posterior: Ensemble, weight: float ) -> None: diff --git a/src/ert/run_models/ensemble_smoother.py b/src/ert/run_models/ensemble_smoother.py index 1e4d13bfed3..659f17431c6 100644 --- a/src/ert/run_models/ensemble_smoother.py +++ b/src/ert/run_models/ensemble_smoother.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from typing import Literal import numpy as np from pydantic import PrivateAttr @@ -23,7 +24,8 @@ logger = logging.getLogger(__name__) -class EnsembleSmootherConfig(InitialEnsembleRunModelConfig, UpdateRunModelConfig): ... +class EnsembleSmootherConfig(InitialEnsembleRunModelConfig, UpdateRunModelConfig): + type: Literal["ensemble_smoother"] = "ensemble_smoother" class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmootherConfig): diff --git a/src/ert/run_models/evaluate_ensemble.py b/src/ert/run_models/evaluate_ensemble.py index 6904c30fe5f..951d6546752 100644 --- a/src/ert/run_models/evaluate_ensemble.py +++ b/src/ert/run_models/evaluate_ensemble.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal from uuid import UUID import numpy as np @@ -17,6 +17,7 @@ class EvaluateEnsembleConfig(RunModelConfig): + type: Literal["evaluate_ensemble"] = "evaluate_ensemble" ensemble_id: str supports_rerunning_failed_realizations: ClassVar[bool] = True diff --git a/src/ert/run_models/manual_update.py b/src/ert/run_models/manual_update.py index 3e2efb749a6..73b38079f90 100644 --- a/src/ert/run_models/manual_update.py +++ b/src/ert/run_models/manual_update.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any +from typing import Any, Literal from uuid import UUID from pydantic import PrivateAttr @@ -17,6 +17,7 @@ class ManualUpdateConfig(UpdateRunModelConfig): + type: Literal["manual_update"] = "manual_update" ensemble_id: str ert_templates: list[tuple[str, str]] diff --git a/src/ert/run_models/manual_update_enif.py b/src/ert/run_models/manual_update_enif.py index f4ecdbadeb5..1ceb29253fa 100644 --- a/src/ert/run_models/manual_update_enif.py +++ b/src/ert/run_models/manual_update_enif.py @@ -2,16 +2,21 @@ import functools import logging +from typing import Literal from ert.analysis import enif_update from ert.storage import Ensemble -from .manual_update import ManualUpdate +from .manual_update import ManualUpdate, ManualUpdateConfig logger = logging.getLogger(__name__) -class ManualUpdateEnIF(ManualUpdate): +class ManualUpdateEnIFConfig(ManualUpdateConfig): + type: Literal["manual_update_enif"] = "manual_update_enif" # type: ignore + + +class ManualUpdateEnIF(ManualUpdateEnIFConfig, ManualUpdate): @classmethod def name(cls) -> str: return "Manual EnIF update (Experimental)" diff --git a/src/ert/run_models/multiple_data_assimilation.py b/src/ert/run_models/multiple_data_assimilation.py index 736aa4b3108..6cb70554eb7 100644 --- a/src/ert/run_models/multiple_data_assimilation.py +++ b/src/ert/run_models/multiple_data_assimilation.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal from uuid import UUID from pydantic import PrivateAttr @@ -31,6 +31,7 @@ class MultipleDataAssimilationConfig( InitialEnsembleRunModelConfig, UpdateRunModelConfig ): + type: Literal["multiple_data_assimilation"] = "multiple_data_assimilation" default_weights: ClassVar[str] = "4, 2, 1" restart_run: bool prior_ensemble_id: str | None diff --git a/src/ert/run_models/single_test_run.py b/src/ert/run_models/single_test_run.py index 9b14a67a174..2e880c517e0 100644 --- a/src/ert/run_models/single_test_run.py +++ b/src/ert/run_models/single_test_run.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Literal + from pydantic import Field from ert.run_models import EnsembleExperiment @@ -10,11 +12,12 @@ class SingleTestRunConfig(EnsembleExperimentConfig): + type: Literal["single_test_run"] = "single_test_run" # type: ignore active_realizations: list[bool] = Field(default_factory=lambda: [True]) minimum_required_realizations: int = 1 -class SingleTestRun(EnsembleExperiment, SingleTestRunConfig): +class SingleTestRun(SingleTestRunConfig, EnsembleExperiment): """ Single test is equivalent to EnsembleExperiment, in that it samples the prior and evaluates it.
There are two key differences:
diff --git a/src/everest/config/everest_config.py b/src/everest/config/everest_config.py index bb876cded74..113d0d797c5 100644 --- a/src/everest/config/everest_config.py +++ b/src/everest/config/everest_config.py @@ -10,6 +10,7 @@ from typing import ( Annotated, Any, + Literal, Optional, Self, TextIO, @@ -179,6 +180,7 @@ def _find_loc(loc: tuple[int | str, ...] | None, file_content: list[str]) -> int class EverestConfig(BaseModelWithContextSupport): + type: Literal["everest_config"] = "everest_config" controls: Annotated[list[ControlConfig], AfterValidator(unique_items)] = Field( description=dedent( """ @@ -1032,7 +1034,7 @@ def result_names(self) -> list[str]: def to_dict(self) -> dict[str, Any]: the_dict = self.model_dump(exclude_none=True, exclude_unset=True) - + the_dict["type"] = self.type if "config_path" in the_dict: the_dict["config_path"] = str(the_dict["config_path"]) diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json index 5e707377626..b4eb910afba 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json @@ -597,5 +597,6 @@ "shape_id": 3 } ], + "type": "ensemble_information_filter", "experiment_type": "Ensemble Information Filter" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json index f15485f8fcb..97a52ee5b89 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json @@ -304,5 +304,6 @@ "shape_id": null } ], + "type": "ensemble_information_filter", "experiment_type": "Ensemble Information Filter" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 2d5e362eb4b..5b44efb65df 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -452,5 +452,6 @@ "shape_id": null } ], + "type": "ensemble_information_filter", "experiment_type": "Ensemble Information Filter" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json index f75dfb9a453..9554f5db754 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json @@ -582,6 +582,7 @@ "shape_id": 3 } ], + "type": "ensemble_experiment", "target_ensemble": "the_experiment", "experiment_type": "Ensemble Experiment" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json index 211970ed822..ca7b9ee9ef7 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json @@ -289,6 +289,7 @@ "shape_id": null } ], + "type": "ensemble_experiment", "target_ensemble": "the_experiment", "experiment_type": "Ensemble Experiment" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 9c13ae3e182..8d0517f3787 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -437,6 +437,7 @@ "shape_id": null } ], + "type": "ensemble_experiment", "target_ensemble": "the_experiment", "experiment_type": "Ensemble Experiment" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json index 768fcf95883..a131a5574a2 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json @@ -597,5 +597,6 @@ "shape_id": 3 } ], + "type": "ensemble_smoother", "experiment_type": "Ensemble Smoother" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json index 30225a99624..14cdbe9077f 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json @@ -304,5 +304,6 @@ "shape_id": null } ], + "type": "ensemble_smoother", "experiment_type": "Ensemble Smoother" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 52c39b2b2ea..fe05c49ed94 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -452,5 +452,6 @@ "shape_id": null } ], + "type": "ensemble_smoother", "experiment_type": "Ensemble Smoother" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json index 5c53671d04c..2a0015093ab 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json @@ -597,6 +597,7 @@ "shape_id": 3 } ], + "type": "multiple_data_assimilation", "restart_run": false, "prior_ensemble_id": null, "weights": "4, 2, 1", diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json index ba8827f04a0..84ce6128bdd 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json @@ -304,6 +304,7 @@ "shape_id": null } ], + "type": "multiple_data_assimilation", "restart_run": false, "prior_ensemble_id": null, "weights": "4, 2, 1", diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index b7c3a0a4e18..f0d6b7f6f14 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -452,6 +452,7 @@ "shape_id": null } ], + "type": "multiple_data_assimilation", "restart_run": false, "prior_ensemble_id": null, "weights": "4, 2, 1", diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json index 6e36d347f23..60dfb5ca3d6 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json @@ -213,6 +213,7 @@ "random_seed": 1, "start_iteration": 0, "minimum_required_realizations": 3, + "type": "evaluate_ensemble", "ensemble_id": "00000000-0000-0000-0000-000000000000", "experiment_type": "Evaluate Ensemble" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json index 578854fdc33..c557fdec5c8 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json @@ -174,6 +174,7 @@ "random_seed": 1, "start_iteration": 0, "minimum_required_realizations": 1, + "type": "evaluate_ensemble", "ensemble_id": "00000000-0000-0000-0000-000000000000", "experiment_type": "Evaluate Ensemble" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index abfdd8b9a66..7fe4f201cfd 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -148,6 +148,7 @@ "random_seed": 1, "start_iteration": 0, "minimum_required_realizations": 3, + "type": "evaluate_ensemble", "ensemble_id": "00000000-0000-0000-0000-000000000000", "experiment_type": "Evaluate Ensemble" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json index e6f8faf5a80..e75f3002eba 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json @@ -228,6 +228,7 @@ }, "auto_scale_observations": [] }, + "type": "manual_update", "ensemble_id": "00000000-0000-0000-0000-000000000000", "ert_templates": [], "experiment_type": "Manual Update" diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json index 81d74a18841..9b71d142bb4 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json @@ -189,6 +189,7 @@ }, "auto_scale_observations": [] }, + "type": "manual_update", "ensemble_id": "00000000-0000-0000-0000-000000000000", "ert_templates": [], "experiment_type": "Manual Update" diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 884f946bb17..5be3cd85679 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -163,6 +163,7 @@ }, "auto_scale_observations": [] }, + "type": "manual_update", "ensemble_id": "00000000-0000-0000-0000-000000000000", "ert_templates": [ [ From 6b9ece311def127454af26d734253ee4c3a4eaa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Wed, 6 May 2026 12:24:19 +0200 Subject: [PATCH 03/11] Remove unused arguments --- src/ert/run_models/model_factory.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/ert/run_models/model_factory.py b/src/ert/run_models/model_factory.py index 746891dc2ff..ecc09fdd748 100644 --- a/src/ert/run_models/model_factory.py +++ b/src/ert/run_models/model_factory.py @@ -310,7 +310,6 @@ def _setup_manual_update( hooked_workflows=config.hooked_workflows, log_path=config.analysis_config.log_path, ert_templates=config.ert_templates, - observations=config.observation_declarations, shape_registry=config.shape_registry, ) return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue) @@ -330,17 +329,12 @@ def _setup_manual_update_enif( ensemble_id=args.ensemble_id, minimum_required_realizations=config.analysis_config.minimum_required_realizations, target_ensemble=args.target_ensemble, - config=config, storage_path=config.ens_path, queue_config=config.queue_config, analysis_settings=config.analysis_config.es_settings, update_settings=update_settings, status_queue=status_queue, runpath_file=config.runpath_file, - design_matrix=config.analysis_config.design_matrix, - parameter_configuration=config.ensemble_config.parameter_configuration, - response_configuration=config.ensemble_config.response_configuration, - derived_response_configuration=config.ensemble_config.derived_response_configuration, ert_templates=config.ert_templates, user_config_file=Path(config.user_config_file), env_vars=config.env_vars, @@ -350,7 +344,6 @@ def _setup_manual_update_enif( substitutions=config.substitutions, hooked_workflows=config.hooked_workflows, log_path=config.analysis_config.log_path, - observations=config.observation_declarations, shape_registry=config.shape_registry, ) From 72eb4dccd45fcd0711b044222ed1261f3d6b12b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Wed, 6 May 2026 12:25:06 +0200 Subject: [PATCH 04/11] Allow experiment_server to run all run models To align everest and ert, we would like to execute all run models through the experiment server, and not have to gui own the run model like it does today. This pr allows the experiment_server to run all run models, and not just the everest run model. --- .../endpoints/experiment_server.py | 80 ++++---- src/ert/run_models/model_factory.py | 137 +++++++------ tests/conftest.py | 51 +++++ tests/ert/conftest.py | 25 --- .../resources/test_run_eclipse_simulator.py | 2 +- .../resources/test_run_flow_simulator.py | 2 +- .../resources/test_run_reservoirsimulator.py | 2 +- tests/ert/unit_tests/resources/test_shell.py | 2 +- .../unit_tests/resources/test_templating.py | 2 +- .../run_models/test_model_factory.py | 30 +-- tests/ert/utils.py | 26 --- tests/everest/conftest.py | 6 + tests/everest/test_everserver.py | 192 +++++++++++++++++- tests/everest/test_templating.py | 2 +- tests/everest/test_yaml_parser.py | 2 +- 15 files changed, 394 insertions(+), 167 deletions(-) diff --git a/src/ert/dark_storage/endpoints/experiment_server.py b/src/ert/dark_storage/endpoints/experiment_server.py index 03a2698d240..5360027d3ba 100644 --- a/src/ert/dark_storage/endpoints/experiment_server.py +++ b/src/ert/dark_storage/endpoints/experiment_server.py @@ -1,6 +1,5 @@ import asyncio import dataclasses -import datetime import logging import os import queue @@ -23,19 +22,23 @@ from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse from fastapi.security import HTTPBasic, HTTPBasicCredentials +from pydantic import TypeAdapter from starlette import status from starlette.requests import Request from starlette.responses import PlainTextResponse, Response from starlette.websockets import WebSocket -from ert.base_model_context import use_runtime_plugins from ert.config import ConfigWarning, QueueSystem from ert.ensemble_evaluator import EndEvent, EvaluatorServerConfig from ert.ensemble_evaluator.event import FullSnapshotEvent, SnapshotUpdateEvent from ert.ensemble_evaluator.snapshot import EnsembleSnapshot -from ert.plugins import get_site_plugins from ert.run_models import StatusEvents -from ert.run_models.everest_run_model import EverestExitCode, EverestRunModel +from ert.run_models.everest_run_model import EverestExitCode +from ert.run_models.model_factory import ( + EverestRunModel, + RunModelConfigs, + _instantiate_run_model, +) from everest.config import EverestConfig from everest.detached.everserver import ( ExperimentState, @@ -218,19 +221,25 @@ async def start_experiment( run_state = ExperimentRunnerState() _runs[run_id] = run_state request_data = await request.json() - # The output of warnings is the task of the user interface, not - # of everserver. Therefore we suppress them here: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ConfigWarning) - config = EverestConfig.with_plugins(request_data) + adapter: TypeAdapter[RunModelConfigs] = TypeAdapter(RunModelConfigs) + config: RunModelConfigs | EverestConfig + if request_data.get("type") == "everest_config": + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConfigWarning) + config = EverestConfig.with_plugins(request_data) + else: + config = adapter.validate_python(request_data) runner = ExperimentRunner(config, run_id) try: background_tasks.add_task(runner.run) - run_state.config_path = config.config_path - - run_state.run_path = config.simulation_dir - run_state.storage_path = config.output_dir - + if isinstance(config, EverestConfig): + run_state.config_path = config.config_file + run_state.run_path = config.output_dir + run_state.storage_path = str(config.storage_dir) + else: + run_state.config_path = config.user_config_file + run_state.run_path = config.runpath_config.runpath_format_string + run_state.storage_path = config.storage_path # Assume client and server is always in the same timezone # so disregard timestamps run_state.start_time_unix = int(time.time()) @@ -328,28 +337,19 @@ async def _get_event(subscriber_id: str, run_id: str) -> StatusEvents: class ExperimentRunner: def __init__( self, - everest_config: EverestConfig, + config: RunModelConfigs, run_id: str, ) -> None: super().__init__() - self._everest_config = everest_config + self._config = config self._run_id = run_id async def run(self) -> None: run = _runs[self._run_id] status_queue: SimpleQueue[StatusEvents] = SimpleQueue() - run_model: EverestRunModel | None = None try: - site_plugins = get_site_plugins() - with use_runtime_plugins(site_plugins): - run_model = EverestRunModel.create( - everest_config=self._everest_config, - experiment_name=f"EnOpt@{datetime.datetime.now().astimezone().isoformat(timespec='seconds')}", - target_ensemble="batch", - status_queue=status_queue, - runtime_plugins=site_plugins, - ) + run_model = _instantiate_run_model(self._config, status_queue) run.status = ExperimentStatus( message="Experiment started", status=ExperimentState.running ) @@ -382,15 +382,20 @@ async def run(self) -> None: await sub.is_done() break await simulation_future - assert run_model.exit_code is not None - exp_status, msg = _get_optimization_status( - run_model.exit_code, - run.events, - ) - run.status = ExperimentStatus( - message=msg, - status=exp_status, - ) + if isinstance(run_model, EverestRunModel): + assert run_model.exit_code is not None + exp_state, msg = _get_optimization_status( + run_model.exit_code, run.events + ) + run_status = ExperimentStatus( + message=msg, + status=exp_state, + ) + else: + run_status = ExperimentStatus( + message="Experiment completed.", status=ExperimentState.completed + ) + run.status = run_status except UserCancelled as e: logging.getLogger(EXPERIMENT_SERVER).info(f"User cancelled: {e}") except Exception as e: @@ -400,7 +405,10 @@ async def run(self) -> None: status=ExperimentState.failed, ) finally: - if run_model and run_model._experiment: + if ( + isinstance(run_model, EverestRunModel) + and run_model._experiment is not None + ): run_model._experiment.status = run.status logging.getLogger(EXPERIMENT_SERVER).info( diff --git a/src/ert/run_models/model_factory.py b/src/ert/run_models/model_factory.py index ecc09fdd748..7e2b553bc99 100644 --- a/src/ert/run_models/model_factory.py +++ b/src/ert/run_models/model_factory.py @@ -1,12 +1,15 @@ from __future__ import annotations import logging +from datetime import datetime from pathlib import Path from queue import SimpleQueue -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated import numpy as np +from pydantic import Field +from ert.base_model_context import use_runtime_plugins from ert.config import ( ConfigValidationError, ConfigWarning, @@ -25,7 +28,9 @@ MANUAL_UPDATE_MODE, TEST_RUN_MODE, ) +from ert.plugins import get_site_plugins from ert.validation import ActiveRange +from everest.config import EverestConfig from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig from .ensemble_information_filter import ( @@ -34,9 +39,10 @@ ) from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig from .evaluate_ensemble import EvaluateEnsemble, EvaluateEnsembleConfig +from .everest_run_model import EverestRunModel from .initial_ensemble_run_model import DictEncodedDataFrame from .manual_update import ManualUpdate, ManualUpdateConfig -from .manual_update_enif import ManualUpdateEnIF +from .manual_update_enif import ManualUpdateEnIF, ManualUpdateEnIFConfig from .multiple_data_assimilation import ( MultipleDataAssimilation, MultipleDataAssimilationConfig, @@ -51,6 +57,18 @@ from ert.run_models.event import StatusEvents +RunModelConfigs = Annotated[ + MultipleDataAssimilationConfig + | EnsembleSmootherConfig + | EnsembleInformationFilterConfig + | SingleTestRunConfig + | EnsembleExperimentConfig + | ManualUpdateConfig + | EvaluateEnsembleConfig + | EverestConfig, + Field(discriminator="type"), +] + logger = logging.getLogger(__name__) @@ -66,31 +84,62 @@ def create_model( "ensemble_size": config.runpath_config.num_realizations, }, ) + runmodel_config = build_run_model_config(config, args) + return _instantiate_run_model(runmodel_config, status_queue) + + +def build_run_model_config(config: ErtConfig, args: Namespace) -> RunModelConfigs: update_settings = config.analysis_config.observation_settings if args.mode == TEST_RUN_MODE: - return _setup_single_test_run(config, args, status_queue) + return _setup_single_test_run(config, args) if args.mode == ENSEMBLE_EXPERIMENT_MODE: - return _setup_ensemble_experiment(config, args, status_queue) + return _setup_ensemble_experiment(config, args) if args.mode == EVALUATE_ENSEMBLE_MODE: - return _setup_evaluate_ensemble(config, args, status_queue) + return _setup_evaluate_ensemble(config, args) if args.mode == ENSEMBLE_SMOOTHER_MODE: - return _setup_ensemble_smoother(config, args, update_settings, status_queue) + return _setup_ensemble_smoother(config, args, update_settings) if args.mode == ENIF_MODE: - return _setup_ensemble_information_filter( - config, args, update_settings, status_queue - ) + return _setup_ensemble_information_filter(config, args, update_settings) if args.mode == ES_MDA_MODE: - return _setup_multiple_data_assimilation( - config, args, update_settings, status_queue - ) + return _setup_multiple_data_assimilation(config, args, update_settings) if args.mode == MANUAL_UPDATE_MODE: - return _setup_manual_update(config, args, update_settings, status_queue) + return _setup_manual_update(config, args, update_settings) if args.mode == MANUAL_ENIF_UPDATE_MODE: - return _setup_manual_update_enif(config, args, update_settings, status_queue) + return _setup_manual_update_enif(config, args, update_settings) raise NotImplementedError(f"Run type not supported {args.mode}") +def _instantiate_run_model( + runmodel_config: RunModelConfigs, + status_queue: SimpleQueue[StatusEvents], +) -> RunModel: + """Instantiate a RunModel from a config object.""" + if isinstance(runmodel_config, EverestConfig): + site_plugins = get_site_plugins() + with use_runtime_plugins(site_plugins): + return EverestRunModel.create( + everest_config=runmodel_config, + experiment_name=f"EnOpt@{datetime.now().astimezone().isoformat(timespec='seconds')}", + target_ensemble="batch", + status_queue=status_queue, + runtime_plugins=site_plugins, + ) + + model_map: dict[str, type[RunModel]] = { + "single_test_run": SingleTestRun, + "ensemble_experiment": EnsembleExperiment, + "evaluate_ensemble": EvaluateEnsemble, + "ensemble_smoother": EnsembleSmoother, + "ensemble_information_filter": EnsembleInformationFilter, + "multiple_data_assimilation": MultipleDataAssimilation, + "manual_update": ManualUpdate, + "manual_update_enif": ManualUpdateEnIF, + } + model_cls = model_map[runmodel_config.type] + return model_cls(**runmodel_config.model_dump(), status_queue=status_queue) + + def _merge_parameters( design_matrix: DesignMatrix | None, parameter_configs: list[ParameterConfig], @@ -116,8 +165,7 @@ def _merge_parameters( def _setup_single_test_run( config: ErtConfig, args: Namespace, - status_queue: SimpleQueue[StatusEvents], -) -> SingleTestRun: +) -> SingleTestRunConfig: experiment_name = ( "single-test-run" if args.experiment_name is None else args.experiment_name ) @@ -132,7 +180,7 @@ def _setup_single_test_run( parameter_configs=config.ensemble_config.parameter_configuration, ) - runmodel_config = SingleTestRunConfig( + return SingleTestRunConfig( random_seed=config.random_seed, runpath_file=config.runpath_file, active_realizations=[True], @@ -158,11 +206,6 @@ def _setup_single_test_run( shape_registry=config.shape_registry, ) - return SingleTestRun( - **runmodel_config.model_dump(), - status_queue=status_queue, - ) - def validate_minimum_realizations( config: ErtConfig, active_realizations: list[bool] @@ -181,8 +224,7 @@ def validate_minimum_realizations( def _setup_ensemble_experiment( config: ErtConfig, args: Namespace, - status_queue: SimpleQueue[StatusEvents], -) -> EnsembleExperiment: +) -> EnsembleExperimentConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) experiment_name = args.experiment_name @@ -193,7 +235,7 @@ def _setup_ensemble_experiment( parameter_configs=config.ensemble_config.parameter_configuration, ) - runmodel_config = EnsembleExperimentConfig( + return EnsembleExperimentConfig( random_seed=config.random_seed, runpath_file=config.runpath_file, active_realizations=active_realizations, @@ -219,20 +261,14 @@ def _setup_ensemble_experiment( shape_registry=config.shape_registry, ) - return EnsembleExperiment( - **runmodel_config.model_dump(), - status_queue=status_queue, - ) - def _setup_evaluate_ensemble( config: ErtConfig, args: Namespace, - status_queue: SimpleQueue[StatusEvents], -) -> EvaluateEnsemble: +) -> EvaluateEnsembleConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) - runmodel_config = EvaluateEnsembleConfig( + return EvaluateEnsembleConfig( random_seed=config.random_seed, active_realizations=active_realizations, ensemble_id=args.ensemble_id, @@ -250,7 +286,6 @@ def _setup_evaluate_ensemble( log_path=config.analysis_config.log_path, shape_registry=config.shape_registry, ) - return EvaluateEnsemble(**runmodel_config.model_dump(), status_queue=status_queue) def _get_and_validate_active_realizations_list( @@ -285,12 +320,11 @@ def _setup_manual_update( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> ManualUpdate: +) -> ManualUpdateConfig: active_realizations = _realizations(args, config.runpath_config.num_realizations) validate_minimum_realizations(config, active_realizations.tolist()) - runmodel_config = ManualUpdateConfig( + return ManualUpdateConfig( random_seed=config.random_seed, active_realizations=active_realizations.tolist(), ensemble_id=args.ensemble_id, @@ -312,18 +346,16 @@ def _setup_manual_update( ert_templates=config.ert_templates, shape_registry=config.shape_registry, ) - return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue) def _setup_manual_update_enif( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> ManualUpdate: +) -> ManualUpdateEnIFConfig: active_realizations = _realizations(args, config.runpath_config.num_realizations) - return ManualUpdateEnIF( + return ManualUpdateEnIFConfig( random_seed=config.random_seed, active_realizations=active_realizations.tolist(), ensemble_id=args.ensemble_id, @@ -333,7 +365,6 @@ def _setup_manual_update_enif( queue_config=config.queue_config, analysis_settings=config.analysis_config.es_settings, update_settings=update_settings, - status_queue=status_queue, runpath_file=config.runpath_file, ert_templates=config.ert_templates, user_config_file=Path(config.user_config_file), @@ -352,8 +383,7 @@ def _setup_ensemble_smoother( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> EnsembleSmoother: +) -> EnsembleSmootherConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) if sum(active_realizations) < 2: @@ -367,7 +397,7 @@ def _setup_ensemble_smoother( require_updateable_param=True, ) - runmodel_config = EnsembleSmootherConfig( + return EnsembleSmootherConfig( target_ensemble=args.target_ensemble, experiment_name=getattr(args, "experiment_name", ""), active_realizations=active_realizations, @@ -394,15 +424,13 @@ def _setup_ensemble_smoother( observations=config.observation_declarations, shape_registry=config.shape_registry, ) - return EnsembleSmoother(**runmodel_config.model_dump(), status_queue=status_queue) def _setup_ensemble_information_filter( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> EnsembleInformationFilter: +) -> EnsembleInformationFilterConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) if sum(active_realizations) < 2: @@ -415,7 +443,7 @@ def _setup_ensemble_information_filter( parameter_configs=config.ensemble_config.parameter_configuration, ) - runmodel_config = EnsembleInformationFilterConfig( + return EnsembleInformationFilterConfig( target_ensemble=args.target_ensemble, experiment_name=getattr(args, "experiment_name", ""), active_realizations=active_realizations, @@ -442,9 +470,6 @@ def _setup_ensemble_information_filter( observations=config.observation_declarations, shape_registry=config.shape_registry, ) - return EnsembleInformationFilter( - **runmodel_config.model_dump(), status_queue=status_queue - ) def _determine_restart_info(args: Namespace) -> tuple[bool, str | None]: @@ -470,8 +495,7 @@ def _setup_multiple_data_assimilation( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> MultipleDataAssimilation: +) -> MultipleDataAssimilationConfig: restart_run, prior_ensemble = _determine_restart_info(args) active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) @@ -486,7 +510,7 @@ def _setup_multiple_data_assimilation( require_updateable_param=True, ) - runmodel_config = MultipleDataAssimilationConfig( + return MultipleDataAssimilationConfig( random_seed=config.random_seed, active_realizations=active_realizations, target_ensemble=_iterative_ensemble_format(args), @@ -516,9 +540,6 @@ def _setup_multiple_data_assimilation( observations=config.observation_declarations, shape_registry=config.shape_registry, ) - return MultipleDataAssimilation( - **runmodel_config.model_dump(), status_queue=status_queue - ) def _realizations( diff --git a/tests/conftest.py b/tests/conftest.py index 8f619fab16f..f477b3b2e96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,39 @@ import os +import shutil +from pathlib import Path from unittest.mock import patch import pytest +from ert.config import ErtConfig from ert.plugins import ErtRuntimePlugins +def source_dir() -> Path: + src = Path("@CMAKE_CURRENT_SOURCE_DIR@/../..") + if src.is_dir(): + return src.relative_to(Path.cwd()) + + # If the file was not correctly configured by cmake, look for the source + # folder, assuming the build folder is inside the source folder. + current_path = Path(__file__) + while current_path != Path("/"): + if (current_path / ".git").is_dir(): + return current_path + # This is to find root dir for git worktrees + elif (current_path / ".git").is_file(): + with (current_path / ".git").open(encoding="utf-8") as f: + for line in f: + if "gitdir:" in line: + return current_path + + current_path = current_path.parent + raise RuntimeError("Cannot find the source folder") + + +SOURCE_DIR: Path = source_dir() + + def pytest_addoption(parser): parser.addoption( "--eclipse-simulator", @@ -138,3 +166,26 @@ def ErtRuntimePluginsWithNoQueueOptions(**kwargs): ErtRuntimePluginsWithNoQueueOptions, ): yield + + +@pytest.fixture(scope="session", name="source_root") +def fixture_source_root(): + return SOURCE_DIR + + +@pytest.fixture(name="setup_case") +def fixture_setup_case(tmp_path_factory, source_root, monkeypatch): + def copy_case(path, config_file): + tmp_path = tmp_path_factory.mktemp(path.replace("/", "-")) + shutil.copytree( + os.path.join(source_root, "test-data/ert", path), tmp_path / "test_data" + ) + monkeypatch.chdir(tmp_path / "test_data") + return ErtConfig.from_file(config_file) + + return copy_case + + +@pytest.fixture +def poly_case(setup_case): + return setup_case("poly_example", "poly.ert") diff --git a/tests/ert/conftest.py b/tests/ert/conftest.py index 00ffd24876e..50d38debae9 100644 --- a/tests/ert/conftest.py +++ b/tests/ert/conftest.py @@ -36,8 +36,6 @@ ) from ert.storage import open_storage -from .utils import SOURCE_DIR - st.register_type_strategy(Path, st.builds(Path, st.text().map(lambda x: "/tmp/" + x))) @@ -97,11 +95,6 @@ def _qt_add_search_paths(qapp): ) -@pytest.fixture(scope="session", name="source_root") -def fixture_source_root(): - return SOURCE_DIR - - @pytest.fixture(scope="class") def class_source_root(request, source_root): request.cls.SOURCE_ROOT = source_root @@ -121,24 +114,6 @@ def maximize_ulimits(): resource.setrlimit(resource.RLIMIT_NOFILE, limits) -@pytest.fixture(name="setup_case") -def fixture_setup_case(tmp_path_factory, source_root, monkeypatch): - def copy_case(path, config_file): - tmp_path = tmp_path_factory.mktemp(path.replace("/", "-")) - shutil.copytree( - os.path.join(source_root, "test-data/ert", path), tmp_path / "test_data" - ) - monkeypatch.chdir(tmp_path / "test_data") - return ErtConfig.from_file(config_file) - - return copy_case - - -@pytest.fixture -def poly_case(setup_case): - return setup_case("poly_example", "poly.ert") - - @pytest.fixture def snake_oil_case_storage(copy_snake_oil_case_storage): with warnings.catch_warnings(): diff --git a/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py b/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py index 453e47543e1..683fd00a4bd 100644 --- a/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py +++ b/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py @@ -8,7 +8,7 @@ import pytest from ert.plugins import ErtPluginManager -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_run_flow_simulator.py b/tests/ert/unit_tests/resources/test_run_flow_simulator.py index 35e233bd3a2..6cc4642bf89 100644 --- a/tests/ert/unit_tests/resources/test_run_flow_simulator.py +++ b/tests/ert/unit_tests/resources/test_run_flow_simulator.py @@ -6,7 +6,7 @@ import pytest -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py b/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py index 7bd5a3cff6e..a48a3272b1c 100644 --- a/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py +++ b/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py @@ -10,7 +10,7 @@ import pytest import resfo -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_shell.py b/tests/ert/unit_tests/resources/test_shell.py index b72de6e2547..e873807343f 100644 --- a/tests/ert/unit_tests/resources/test_shell.py +++ b/tests/ert/unit_tests/resources/test_shell.py @@ -11,7 +11,7 @@ from ert.config import ErtConfig from ert.config.workflow_job import ExecutableWorkflow from ert.plugins import get_site_plugins -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_templating.py b/tests/ert/unit_tests/resources/test_templating.py index 6ac8b3db2b1..c09b9ce0800 100644 --- a/tests/ert/unit_tests/resources/test_templating.py +++ b/tests/ert/unit_tests/resources/test_templating.py @@ -7,7 +7,7 @@ import jinja2 import pytest -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/run_models/test_model_factory.py b/tests/ert/unit_tests/run_models/test_model_factory.py index 9e4e218719f..d27e476a7ef 100644 --- a/tests/ert/unit_tests/run_models/test_model_factory.py +++ b/tests/ert/unit_tests/run_models/test_model_factory.py @@ -28,6 +28,7 @@ model_factory, ) from ert.run_models.model_factory import ( + _instantiate_run_model, _setup_ensemble_information_filter, _setup_ensemble_smoother, _setup_multiple_data_assimilation, @@ -123,7 +124,7 @@ def test_custom_realizations(): def test_setup_single_test_run(tmp_path): - model = model_factory._setup_single_test_run( + config = model_factory._setup_single_test_run( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( current_ensemble="current-ensemble", @@ -131,14 +132,14 @@ def test_setup_single_test_run(tmp_path): random_seed=None, experiment_name=None, ), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, SingleTestRun) assert model._storage.path == tmp_path def test_setup_single_test_run_with_ensemble(tmp_path): - model = model_factory._setup_single_test_run( + config = model_factory._setup_single_test_run( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( current_ensemble="current-ensemble", @@ -146,14 +147,14 @@ def test_setup_single_test_run_with_ensemble(tmp_path): random_seed=None, experiment_name=None, ), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, SingleTestRun) assert model._storage.path == tmp_path def test_setup_ensemble_experiment(tmp_path): - model = model_factory._setup_ensemble_experiment( + config = model_factory._setup_ensemble_experiment( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( realizations=None, @@ -162,8 +163,8 @@ def test_setup_ensemble_experiment(tmp_path): target_ensemble=None, experiment_name="ensemble_experiment", ), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, EnsembleExperiment) assert model.active_realizations == [True] * 100 @@ -171,7 +172,7 @@ def test_setup_ensemble_experiment(tmp_path): @pytest.mark.filterwarnings("ignore:MIN_REALIZATIONS") def test_setup_ensemble_smoother(tmp_path): - model = model_factory._setup_ensemble_smoother( + config = model_factory._setup_ensemble_smoother( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( realizations="0-4,7,8", @@ -180,8 +181,8 @@ def test_setup_ensemble_smoother(tmp_path): experiment_name="just_smoothing", ), ObservationSettings(), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, EnsembleSmoother) assert ( model.active_realizations @@ -191,7 +192,7 @@ def test_setup_ensemble_smoother(tmp_path): @pytest.mark.filterwarnings("ignore:MIN_REALIZATIONS") def test_setup_multiple_data_assimilation(tmp_path): - model = model_factory._setup_multiple_data_assimilation( + config = model_factory._setup_multiple_data_assimilation( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( realizations="0-4,8", @@ -203,8 +204,8 @@ def test_setup_multiple_data_assimilation(tmp_path): starting_iteration=0, ), ObservationSettings(), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, MultipleDataAssimilation) assert model.weights == "6,4,2" assert model._parsed_weights == MultipleDataAssimilation.parse_weights("6,4,2") @@ -269,9 +270,10 @@ def test_multiple_data_assimilation_restart_paths( with patch( "ert.run_models.run_model.Storage.get_ensemble", return_value=ensemble_mock ): - model = model_factory._setup_multiple_data_assimilation( - config, args, ObservationSettings(), queue.SimpleQueue() + config = model_factory._setup_multiple_data_assimilation( + config, args, ObservationSettings() ) + model = _instantiate_run_model(config, queue.SimpleQueue()) base_path = tmp_path / "simulations" expected_path = [str(base_path / expected) for expected in expected_path] assert set(model.paths) == set(expected_path) @@ -299,7 +301,7 @@ def test_num_realizations_specified_incorrectly_raises(analysis_mode): ConfigValidationError, match="Number of active realizations must be at least 2 for an update step", ): - analysis_mode(config, args, ObservationSettings(), queue.SimpleQueue()) + analysis_mode(config, args, ObservationSettings()) @pytest.mark.parametrize( @@ -372,4 +374,4 @@ def test_that_setting_up_experiment_with_update_step_raises_config_validation_er ConfigValidationError, match="Number of active realizations must be at least 2 for an update step", ): - experiment_setup_method(config, args, MagicMock(), MagicMock()) + experiment_setup_method(config, args, MagicMock()) diff --git a/tests/ert/utils.py b/tests/ert/utils.py index 0b2f642826a..14f15d9f74f 100644 --- a/tests/ert/utils.py +++ b/tests/ert/utils.py @@ -5,7 +5,6 @@ import time import uuid from collections.abc import Callable -from pathlib import Path from typing import TYPE_CHECKING, Self import zmq @@ -29,31 +28,6 @@ from ert.scheduler.driver import Driver -def source_dir() -> Path: - src = Path("@CMAKE_CURRENT_SOURCE_DIR@/../..") - if src.is_dir(): - return src.relative_to(Path.cwd()) - - # If the file was not correctly configured by cmake, look for the source - # folder, assuming the build folder is inside the source folder. - current_path = Path(__file__) - while current_path != Path("/"): - if (current_path / ".git").is_dir(): - return current_path - # This is to find root dir for git worktrees - elif (current_path / ".git").is_file(): - with (current_path / ".git").open(encoding="utf-8") as f: - for line in f: - if "gitdir:" in line: - return current_path - - current_path = current_path.parent - raise RuntimeError("Cannot find the source folder") - - -SOURCE_DIR: Path = source_dir() - - def wait_until(func, interval=0.5, timeout=30): """Waits until func returns True. diff --git a/tests/everest/conftest.py b/tests/everest/conftest.py index bd44a901c0a..3818193d599 100644 --- a/tests/everest/conftest.py +++ b/tests/everest/conftest.py @@ -14,6 +14,7 @@ import yaml from ert.base_model_context import use_runtime_plugins +from ert.config import ErtConfig from ert.config.queue_config import LocalQueueOptions, LsfQueueOptions from ert.ensemble_evaluator import EvaluatorServerConfig from ert.plugins import ErtRuntimePlugins, get_site_plugins @@ -172,6 +173,11 @@ def copy_math_func_test_data_to_tmp(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) +@pytest.fixture +def poly_ert_config(poly_case) -> ErtConfig: + return ErtConfig.from_file("poly.ert") + + @pytest.fixture def cached_example(pytestconfig): cache = pytestconfig.cache diff --git a/tests/everest/test_everserver.py b/tests/everest/test_everserver.py index 14925264d4f..d6cf422cb43 100644 --- a/tests/everest/test_everserver.py +++ b/tests/everest/test_everserver.py @@ -13,8 +13,14 @@ from ert.config import ConfigWarning from ert.dark_storage.app import app -from ert.dark_storage.endpoints.experiment_server import ExperimentRunnerState, _runs +from ert.dark_storage.endpoints.experiment_server import ( + ExperimentRunner, + ExperimentRunnerState, + _runs, +) from ert.ensemble_evaluator import EndEvent +from ert.namespace import Namespace +from ert.run_models.model_factory import build_run_model_config from ert.scheduler.event import FinishedEvent from ert.services import create_ertserver_client from ert.storage import ExperimentState @@ -30,8 +36,13 @@ from everest.strings import ( OPT_FAILURE_ALL_REALIZATIONS, OPT_FAILURE_REALIZATIONS, + EverEndpoints, ) +_FAKE_ENSEMBLE_ID = "00000000-0000-0000-0000-000000000001" + +_START_EXPERIMENT_URL = f"/experiment_server/{EverEndpoints.start_experiment}" + @pytest.fixture def setup_client(monkeypatch): @@ -377,3 +388,182 @@ def receive_event(): result.append(await receive_task) assert result == [jsonable_encoder(expected_result)] + + +@pytest.fixture +def start_experiment_client(monkeypatch): + original = dict(_runs) + monkeypatch.setenv("ERT_STORAGE_TOKEN", "password") + client = TestClient(app) + credentials = b64encode(b"username:password").decode() + yield client, {"Authorization": f"Basic {credentials}"} + _runs.clear() + _runs.update(original) + + +@pytest.mark.parametrize( + ("mode", "extra_kwargs"), + [ + pytest.param( + "test_run", + { + "experiment_name": None, + "current_ensemble": "prior", + "realizations": None, + }, + id="single_test_run", + ), + pytest.param( + "ensemble_experiment", + { + "experiment_name": "exp", + "current_ensemble": "prior", + "realizations": None, + }, + id="ensemble_experiment", + ), + pytest.param( + "ensemble_smoother", + { + "experiment_name": "exp", + "target_ensemble": "prior_%d", + "realizations": None, + }, + id="ensemble_smoother", + ), + pytest.param( + "ensemble_information_filter", + { + "experiment_name": "exp", + "target_ensemble": "prior_%d", + "realizations": None, + }, + id="ensemble_information_filter", + ), + pytest.param( + "es_mda", + { + "experiment_name": "exp", + "target_ensemble": "prior_%d", + "weights": "4, 2, 1", + "restart_ensemble_id": None, + "realizations": None, + }, + id="multiple_data_assimilation", + ), + pytest.param( + "manual_update", + { + "ensemble_id": _FAKE_ENSEMBLE_ID, + "target_ensemble": "prior_1", + "realizations": None, + }, + id="manual_update", + ), + pytest.param( + "evaluate_ensemble", + {"ensemble_id": _FAKE_ENSEMBLE_ID, "realizations": None}, + id="evaluate_ensemble", + ), + ], +) +def test_that_start_experiment_accepts_all_ert_run_model_config_types( + start_experiment_client, poly_ert_config, mode, extra_kwargs +): + client, auth_headers = start_experiment_client + run_model_config = build_run_model_config( + poly_ert_config, Namespace(mode=mode, **extra_kwargs) + ) + payload = run_model_config.model_dump(mode="json") + + with patch.object(ExperimentRunner, "run", new_callable=AsyncMock) as mock_run: + response = client.post( + _START_EXPERIMENT_URL, json=payload, headers=auth_headers + ) + mock_run.assert_called_once() + assert response.status_code == 200 + body = response.json() + assert "run_id" in body + assert body["run_id"] in _runs + + +def test_that_start_experiment_sets_paths_from_ert_config( + start_experiment_client, poly_ert_config +): + client, auth_headers = start_experiment_client + run_model_config = build_run_model_config( + poly_ert_config, + Namespace( + mode="ensemble_experiment", + experiment_name="exp", + current_ensemble="prior", + realizations=None, + ), + ) + payload = run_model_config.model_dump(mode="json") + + with patch.object(ExperimentRunner, "run", new_callable=AsyncMock): + response = client.post( + _START_EXPERIMENT_URL, json=payload, headers=auth_headers + ) + + assert response.status_code == 200 + run_id = response.json()["run_id"] + run_state = _runs[run_id] + assert str(run_state.config_path) == str(run_model_config.user_config_file) + assert run_state.run_path == run_model_config.runpath_config.runpath_format_string + assert run_state.storage_path == run_model_config.storage_path + assert run_state.start_time_unix is not None + + +def test_that_start_experiment_with_invalid_ert_payload_returns_an_error_response(): + client = TestClient(app, raise_server_exceptions=False) + credentials = b64encode(b"username:password").decode() + auth_headers = {"Authorization": f"Basic {credentials}"} + response = client.post( + _START_EXPERIMENT_URL, + json={"type": "not_a_valid_type"}, + headers=auth_headers, + ) + assert response.status_code >= 400 + + +def test_that_start_experiment_requires_authentication( + start_experiment_client, poly_ert_config +): + client, _ = start_experiment_client + run_model_config = build_run_model_config( + poly_ert_config, + Namespace( + mode="ensemble_experiment", + experiment_name="exp", + current_ensemble="prior", + realizations=None, + ), + ) + response = client.post( + _START_EXPERIMENT_URL, json=run_model_config.model_dump(mode="json") + ) + assert response.status_code == 401 + + +def test_that_start_experiment_rejects_wrong_password( + start_experiment_client, poly_ert_config +): + client, _ = start_experiment_client + wrong_credentials = b64encode(b"username:wrong_password").decode() + run_model_config = build_run_model_config( + poly_ert_config, + Namespace( + mode="ensemble_experiment", + experiment_name="exp", + current_ensemble="prior", + realizations=None, + ), + ) + response = client.post( + _START_EXPERIMENT_URL, + json=run_model_config.model_dump(mode="json"), + headers={"Authorization": f"Basic {wrong_credentials}"}, + ) + assert response.status_code == 401 diff --git a/tests/everest/test_templating.py b/tests/everest/test_templating.py index a70c4b80dac..47e852aa886 100644 --- a/tests/everest/test_templating.py +++ b/tests/everest/test_templating.py @@ -12,8 +12,8 @@ from ert.plugins import get_site_plugins from ert.run_models.everest_run_model import EverestRunModel from everest.config import EverestConfig, InstallTemplateConfig +from tests.conftest import SOURCE_DIR from tests.ert.unit_tests.resources._import_from_location import import_from_location -from tests.ert.utils import SOURCE_DIR from tests.everest.utils import everest_config_with_defaults CONFIG = { diff --git a/tests/everest/test_yaml_parser.py b/tests/everest/test_yaml_parser.py index 0a9f73ce91a..ff7c2b64325 100644 --- a/tests/everest/test_yaml_parser.py +++ b/tests/everest/test_yaml_parser.py @@ -24,7 +24,7 @@ def test_read_file(tmp_path, monkeypatch): encoding="utf-8", ) everest_config = EverestConfig.load_file("config.yml") - keys = ["config_path", "controls", "model", "objective_functions"] + keys = ["config_path", "controls", "model", "objective_functions", "type"] assert sorted(everest_config.to_dict().keys()) == sorted(keys) exp_dir, exp_fn = os.path.split(os.path.realpath("config.yml")) From 4a395f3d8252c9a3324c3bf30c78b45c656dabae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Wed, 6 May 2026 20:11:55 +0200 Subject: [PATCH 05/11] Move ownership of run_model from gui to server --- .../endpoints/experiment_server.py | 154 +++++++++- src/ert/gui/experiments/experiment_client.py | 69 ++++- src/ert/gui/experiments/experiment_panel.py | 189 +++++++------ src/everest/gui/main_window.py | 5 +- src/everest/strings.py | 4 + .../test_experiment_server_new_endpoints.py | 267 ++++++++++++++++++ 6 files changed, 581 insertions(+), 107 deletions(-) create mode 100644 tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py diff --git a/src/ert/dark_storage/endpoints/experiment_server.py b/src/ert/dark_storage/endpoints/experiment_server.py index 5360027d3ba..3045ca68eee 100644 --- a/src/ert/dark_storage/endpoints/experiment_server.py +++ b/src/ert/dark_storage/endpoints/experiment_server.py @@ -10,7 +10,7 @@ import warnings from base64 import b64decode from queue import SimpleQueue -from typing import Annotated +from typing import TYPE_CHECKING, Annotated from fastapi import ( APIRouter, @@ -51,6 +51,9 @@ EverEndpoints, ) +if TYPE_CHECKING: + from ert.run_models.run_model import RunModel + router = APIRouter(prefix="/experiment_server", tags=["experiment_server"]) @@ -67,6 +70,9 @@ class ExperimentRunnerState: run_path: str | os.PathLike[str] | None = None storage_path: str | os.PathLike[str] | None = None start_time_unix: int | None = None + run_model: "RunModel | None" = dataclasses.field(default=None) + supports_rerunning_failed_realizations: bool = False + has_failed_realizations: bool = False _runs: dict[str, ExperimentRunnerState] = {} @@ -243,7 +249,14 @@ async def start_experiment( # Assume client and server is always in the same timezone # so disregard timestamps run_state.start_time_unix = int(time.time()) - return JSONResponse({"run_id": run_id}) + return JSONResponse( + { + "run_id": run_id, + "supports_rerunning_failed_realizations": ( + run_state.supports_rerunning_failed_realizations + ), + } + ) except Exception as e: run_state.status = ExperimentStatus( status=ExperimentState.failed, @@ -290,6 +303,112 @@ async def start_time( return Response(str(run.start_time_unix), status_code=200) +@router.post("/" + EverEndpoints.check_runpath) +async def check_runpath( + request: Request, + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> JSONResponse: + _log(request) + _check_user(credentials) + status_queue: SimpleQueue[StatusEvents] = SimpleQueue() + request_data = await request.json() + adapter: TypeAdapter[RunModelConfigs] = TypeAdapter(RunModelConfigs) + try: + config = adapter.validate_python(request_data) + run_model = _instantiate_run_model(config, status_queue) + try: + runpath_exists = run_model.check_if_runpath_exists() + return JSONResponse( + { + "runpath_exists": runpath_exists, + "num_existing": run_model.get_number_of_existing_runpaths(), + "num_active": run_model.get_number_of_active_realizations(), + } + ) + finally: + run_model._storage.close() + except Exception as e: + raise HTTPException( + status_code=422, detail=f"Could not check runpath: {e!s}" + ) from e + + +@router.post("/" + EverEndpoints.delete_runpaths) +async def delete_runpaths( + request: Request, + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> Response: + _log(request) + _check_user(credentials) + status_queue: SimpleQueue[StatusEvents] = SimpleQueue() + request_data = await request.json() + adapter: TypeAdapter[RunModelConfigs] = TypeAdapter(RunModelConfigs) + try: + config = adapter.validate_python(request_data) + run_model = _instantiate_run_model(config, status_queue) + try: + run_model.rm_run_path() + finally: + run_model._storage.close() + return Response("Runpaths deleted.", 200) + except Exception as e: + raise HTTPException( + status_code=422, detail=f"Could not delete runpaths: {e!s}" + ) from e + + +@router.post(f"/{EverEndpoints.rerun_failed}/{{run_id}}") +async def rerun_failed( + request: Request, + run: Annotated[ExperimentRunnerState, Depends(_get_run)], + run_id: str, + background_tasks: BackgroundTasks, + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> JSONResponse: + _log(request) + _check_user(credentials) + if run.run_model is None: + raise HTTPException( + status_code=400, detail=f"Run '{run_id}' has no run model to rerun." + ) + if not run.supports_rerunning_failed_realizations: + raise HTTPException( + status_code=400, + detail=f"Run '{run_id}' does not support rerunning failed realizations.", + ) + new_run_id = str(uuid.uuid4()) + new_run_state = ExperimentRunnerState( + config_path=run.config_path, + run_path=run.run_path, + storage_path=run.storage_path, + run_model=run.run_model, + supports_rerunning_failed_realizations=run.supports_rerunning_failed_realizations, + ) + _runs[new_run_id] = new_run_state + runner = ExperimentRunner(None, new_run_id) + background_tasks.add_task(runner.run, rerun=True) + new_run_state.start_time_unix = int(time.time()) + return JSONResponse( + { + "new_run_id": new_run_id, + "supports_rerunning_failed_realizations": ( + new_run_state.supports_rerunning_failed_realizations + ), + } + ) + + +@router.get(f"/{EverEndpoints.has_failed_realizations}/{{run_id}}") +def has_failed_realizations_endpoint( + request: Request, + run: Annotated[ExperimentRunnerState, Depends(_get_run)], + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> JSONResponse: + _log(request) + _check_user(credentials) + return JSONResponse({"has_failed": run.has_failed_realizations}) + + @router.websocket(f"/{EverEndpoints.events}/{{run_id}}") async def websocket_endpoint(websocket: WebSocket, run_id: str) -> None: await websocket.accept() @@ -337,7 +456,7 @@ async def _get_event(subscriber_id: str, run_id: str) -> StatusEvents: class ExperimentRunner: def __init__( self, - config: RunModelConfigs, + config: RunModelConfigs | None, run_id: str, ) -> None: super().__init__() @@ -345,11 +464,24 @@ def __init__( self._config = config self._run_id = run_id - async def run(self) -> None: + async def run(self, rerun: bool = False) -> None: run = _runs[self._run_id] status_queue: SimpleQueue[StatusEvents] = SimpleQueue() + run_model: RunModel | None = None try: - run_model = _instantiate_run_model(self._config, status_queue) + if rerun and run.run_model is not None: + run_model = run.run_model + # Rewire status_queue so events go to this run's event loop + run_model._status_queue = status_queue + else: + assert self._config is not None, ( + "ExperimentRunner.run() called without config for a fresh run" + ) + run_model = _instantiate_run_model(self._config, status_queue) + run.run_model = run_model + run.supports_rerunning_failed_realizations = ( + run_model.supports_rerunning_failed_realizations + ) run.status = ExperimentStatus( message="Experiment started", status=ExperimentState.running ) @@ -405,11 +537,13 @@ async def run(self) -> None: status=ExperimentState.failed, ) finally: - if ( - isinstance(run_model, EverestRunModel) - and run_model._experiment is not None - ): - run_model._experiment.status = run.status + if run_model is not None: + run.has_failed_realizations = run_model.has_failed_realizations() + if ( + isinstance(run_model, EverestRunModel) + and run_model._experiment is not None + ): + run_model._experiment.status = run.status logging.getLogger(EXPERIMENT_SERVER).info( f"ExperimentRunner done. Items left in queue: {status_queue.qsize()}" diff --git a/src/ert/gui/experiments/experiment_client.py b/src/ert/gui/experiments/experiment_client.py index e389066a42a..a92ddd38372 100644 --- a/src/ert/gui/experiments/experiment_client.py +++ b/src/ert/gui/experiments/experiment_client.py @@ -7,7 +7,7 @@ import traceback from base64 import b64encode from http import HTTPStatus -from pathlib import Path +from typing import Any import requests from pydantic import ValidationError @@ -27,12 +27,12 @@ class ExperimentClient: def __init__( self, - run_id: str, url: str, cert_file: str, username: str, password: str, ssl_context: ssl.SSLContext, + run_id: str | None = None, ) -> None: self._run_id = run_id self._url = url @@ -60,6 +60,15 @@ def _http_post(self, endpoint: str) -> requests.Response: proxies={"http": None, "https": None}, # type: ignore ) + def _http_post_json(self, endpoint: str, body: dict[str, Any]) -> requests.Response: + return requests.post( + f"{self._url}/{endpoint}", + json=body, + verify=self._cert, + auth=(self._username, self._password), + proxies={"http": None, "https": None}, # type: ignore + ) + @property def config(self) -> dict[str, str]: return self._http_get(f"{EverEndpoints.config_path}/{self._run_id}").json() @@ -68,15 +77,51 @@ def config(self) -> dict[str, str]: def credentials(self) -> str: return b64encode(f"{self._username}:{self._password}".encode()).decode() + def check_runpath(self, config_json: dict[str, Any]) -> dict[str, Any]: + response = self._http_post_json(EverEndpoints.check_runpath, config_json) + response.raise_for_status() + return response.json() + + def delete_runpaths(self, config_json: dict[str, Any]) -> None: + response = self._http_post_json(EverEndpoints.delete_runpaths, config_json) + response.raise_for_status() + + def start_experiment(self, config_json: dict[str, Any]) -> tuple[str, bool]: + response = self._http_post_json(EverEndpoints.start_experiment, config_json) + response.raise_for_status() + data = response.json() + self._run_id = data["run_id"] + return self._run_id, data.get("supports_rerunning_failed_realizations", False) + + def rerun_failed(self) -> tuple[str, bool]: + assert self._run_id is not None, "No active run to rerun" + response = self._http_post(f"{EverEndpoints.rerun_failed}/{self._run_id}") + response.raise_for_status() + data = response.json() + self._run_id = data["new_run_id"] + return self._run_id, data.get("supports_rerunning_failed_realizations", False) + + def has_failed_realizations(self) -> bool: + assert self._run_id is not None, "No active run" + response = self._http_get( + f"{EverEndpoints.has_failed_realizations}/{self._run_id}" + ) + response.raise_for_status() + return bool(response.json().get("has_failed", False)) + def setup_event_queue_from_ws_endpoint( self, + event_queue: queue.SimpleQueue[StatusEvents] | None = None, refresh_interval: float = 0.01, open_timeout: float = 30, websocket_recv_timeout: float = 1.0, ) -> tuple[queue.SimpleQueue[StatusEvents], ErtThread]: - event_queue: queue.SimpleQueue[StatusEvents] = queue.SimpleQueue() + if event_queue is None: + event_queue = queue.SimpleQueue() + out_queue = event_queue def passthrough_ws_events() -> None: + assert self._run_id is not None, "No active run to stream events for" try: with connect( self._url.replace("https://", "wss://") @@ -93,7 +138,7 @@ def passthrough_ws_events() -> None: if message: try: event = status_event_from_json(message) - event_queue.put(event) + out_queue.put(event) except ValidationError as e: logger.error( "Error when processing event %s", exc_info=e @@ -106,14 +151,18 @@ def passthrough_ws_events() -> None: logger.debug(traceback.format_exc()) monitor_thread = ErtThread( - name="everest_gui_event_monitor", + name="ert_gui_event_monitor", target=passthrough_ws_events, daemon=True, ) - return event_queue, monitor_thread + return out_queue, monitor_thread - def create_run_model_api(self) -> RunModelAPI: + def create_run_model_api( + self, + experiment_name: str, + supports_rerunning_failed_realizations: bool, + ) -> RunModelAPI: def start_fn( evaluator_server_config: EvaluatorServerConfig, rerun_failed_realizations: bool = False, @@ -121,11 +170,11 @@ def start_fn( pass return RunModelAPI( - experiment_name=Path(self.config["config_path"]).name, - supports_rerunning_failed_realizations=False, + experiment_name=experiment_name, + supports_rerunning_failed_realizations=supports_rerunning_failed_realizations, start_simulations_thread=start_fn, cancel=self.stop, - has_failed_realizations=lambda: False, + has_failed_realizations=self.has_failed_realizations, ) def stop(self) -> None: diff --git a/src/ert/gui/experiments/experiment_panel.py b/src/ert/gui/experiments/experiment_panel.py index 4683ba200a2..1e68e68c57e 100644 --- a/src/ert/gui/experiments/experiment_panel.py +++ b/src/ert/gui/experiments/experiment_panel.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json +import ssl from collections import OrderedDict from pathlib import Path from queue import SimpleQueue @@ -11,7 +13,6 @@ from PyQt6.QtWidgets import ( QApplication, QCheckBox, - QDialog, QFrame, QHBoxLayout, QMessageBox, @@ -22,27 +23,25 @@ QWidget, ) -from _ert.threading import ErtThread -from ert.config import QueueSystem -from ert.ensemble_evaluator import EvaluatorServerConfig from ert.gui.detect_mode import is_dark_mode from ert.gui.ertnotifier import ErtNotifier from ert.gui.find_ert_info import find_ert_info from ert.gui.icon_utils import load_icon from ert.gui.summarypanel import SummaryPanel -from ert.run_models import RunModel, StatusEvents, create_model +from ert.run_models import RunModel, RunModelAPI, StatusEvents +from ert.run_models.model_factory import build_run_model_config from .combobox_with_description import QComboBoxWithDescription from .ensemble_experiment_panel import EnsembleExperimentPanel from .ensemble_information_filter_panel import EnsembleInformationFilterPanel from .ensemble_smoother_panel import EnsembleSmootherPanel from .evaluate_ensemble_panel import EvaluateEnsemblePanel +from .experiment_client import ExperimentClient from .experiment_config_panel import ExperimentConfigPanel from .manual_update_panel import ManualUpdatePanel from .multiple_data_assimilation_panel import MultipleDataAssimilationPanel from .run_dialog import RunDialog from .single_test_run_panel import SingleTestRunPanel -from .view.runpath_progress_widget import RunpathProgressWidget if TYPE_CHECKING: from ert.config import ErtConfig @@ -58,20 +57,6 @@ def create_md_table(kv: dict[str, str], output: str) -> str: return output -def get_simulation_thread( - model: Any, rerun_failed_realizations: bool = False, use_ipc_protocol: bool = False -) -> ErtThread: - evaluator_server_config = EvaluatorServerConfig(use_ipc_protocol=use_ipc_protocol) - - def run() -> None: - model.api.start_simulations_thread( - evaluator_server_config=evaluator_server_config, - rerun_failed_realizations=rerun_failed_realizations, - ) - - return ErtThread(name="ert_gui_simulation_thread", target=run, daemon=True) - - class ExperimentPanel(QWidget): experiment_type_changed = Signal(ExperimentConfigPanel) experiment_started = Signal(RunDialog) @@ -296,35 +281,59 @@ def get_experiment_arguments(self) -> Any: simulation_widget = self._experiment_widgets[self.get_current_experiment_type()] return simulation_widget.get_experiment_arguments() + def _create_experiment_client(self) -> ExperimentClient: + conn_info_path = Path(self.config.ens_path) / "storage_server.json" + conn_info: dict[str, Any] = json.loads( + conn_info_path.read_text(encoding="utf-8") + ) + url = conn_info["urls"][0] + cert_file = conn_info["cert"] + ssl_context = ssl.create_default_context() + ssl_context.load_verify_locations(cafile=cert_file) + return ExperimentClient( + url=f"{url}/experiment_server", + cert_file=cert_file, + username="__token__", + password=conn_info["authtoken"], + ssl_context=ssl_context, + ) + def run_experiment(self) -> None: args = self.get_experiment_arguments() QApplication.setOverrideCursor(Qt.CursorShape.WaitCursor) - event_queue: SimpleQueue[StatusEvents] = SimpleQueue() try: - model = create_model( - self.config, - args, - event_queue, + run_model_config = build_run_model_config(self.config, args) + config_json = run_model_config.model_dump(mode="json") + client = self._create_experiment_client() + except Exception as e: + QApplication.restoreOverrideCursor() + QMessageBox.warning( + self, + "ERROR: Failed to prepare experiment", + str(e), + QMessageBox.StandardButton.Ok, ) + return + + QApplication.restoreOverrideCursor() - except ValueError as e: + try: + runpath_check = client.check_runpath(config_json) + except Exception as e: QMessageBox.warning( self, - "ERROR: Failed to create experiment", + "ERROR: run_path check failed", str(e), QMessageBox.StandardButton.Ok, ) return - self._model = model - - QApplication.restoreOverrideCursor() - if model.check_if_runpath_exists(): + if runpath_check.get("runpath_exists"): + num_existing = runpath_check.get("num_existing", 0) + num_active = runpath_check.get("num_active", 0) msg_box = QMessageBox(self) msg_box.setObjectName("RUN_PATH_WARNING_BOX") - msg_box.setIcon(QMessageBox.Icon.Warning) - msg_box.setText("Run experiments") msg_box.setInformativeText( "ERT is running in an existing runpath.\n\n" @@ -333,55 +342,28 @@ def run_experiment(self) -> None: "might be overwritten.\n" "- Previously generated files might " "be used if not configured correctly.\n" - f"- {model.get_number_of_existing_runpaths()} out " - f"of {model.get_number_of_active_realizations()} realizations " + f"- {num_existing} out of {num_active} realizations " "are running in existing runpaths.\n" "Are you sure you want to continue?" ) - delete_runpath_checkbox = QCheckBox() delete_runpath_checkbox.setText("Delete run_path") msg_box.setCheckBox(delete_runpath_checkbox) - msg_box.setStandardButtons( QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No ) msg_box.setDefaultButton(QMessageBox.StandardButton.No) - msg_box.setWindowModality(Qt.WindowModality.ApplicationModal) - msg_box_res = msg_box.exec() - if msg_box_res == QMessageBox.StandardButton.No: - self._model._storage.close() + if msg_box.exec() == QMessageBox.StandardButton.No: return if delete_runpath_checkbox.checkState() == Qt.CheckState.Checked: - progress_dialog = QDialog(self) - progress_dialog.setObjectName("RUN_PATH_PROGRESS_DIALOG") - progress_dialog.setWindowTitle("Deleting runpaths") - progress_dialog.setWindowModality(Qt.WindowModality.ApplicationModal) - progress_layout = QVBoxLayout(progress_dialog) - progress_layout.setContentsMargins(0, 0, 0, 0) - - progress_widget = RunpathProgressWidget( - progress_dialog, - initial_status_text="Deleting runpaths...", - completed_action="deleted", - ) - progress_layout.addWidget(progress_widget) - progress_dialog.resize(420, 120) - progress_dialog.show() - QApplication.processEvents() - + QApplication.setOverrideCursor(Qt.CursorShape.WaitCursor) try: - model.rm_run_path( - progress_tracker=progress_widget, - # Force UI update during long deletion process - progress_callback=QApplication.processEvents, - ) + client.delete_runpaths(config_json) except OSError as e: - progress_dialog.close() - progress_dialog.deleteLater() + QApplication.restoreOverrideCursor() msg_box = QMessageBox(self) msg_box.setObjectName("RUN_PATH_ERROR_BOX") msg_box.setIcon(QMessageBox.Icon.Warning) @@ -394,20 +376,50 @@ def run_experiment(self) -> None: ) msg_box.setDefaultButton(QMessageBox.StandardButton.No) msg_box.setWindowModality(Qt.WindowModality.ApplicationModal) - msg_box_res = msg_box.exec() - if msg_box_res == QMessageBox.StandardButton.No: + if msg_box.exec() == QMessageBox.StandardButton.No: return else: - progress_dialog.close() - progress_dialog.deleteLater() + QApplication.restoreOverrideCursor() self.configuration_summary.log_summary( - args.mode, model.get_number_of_active_realizations() + args.mode, runpath_check.get("num_active", 0) + ) + + try: + _run_id, supports_rerunning = client.start_experiment(config_json) + except Exception as e: + QMessageBox.warning( + self, + "ERROR: Failed to start experiment", + str(e), + QMessageBox.StandardButton.Ok, + ) + return + + event_queue: SimpleQueue[StatusEvents] = SimpleQueue() + _, monitor_thread = client.setup_event_queue_from_ws_endpoint( + event_queue=event_queue + ) + + experiment_name = self.get_current_experiment_type().display_name() + + def start_fn( + evaluator_server_config: Any = None, + rerun_failed_realizations: bool = False, + ) -> None: + pass + + run_model_api = RunModelAPI( + experiment_name=experiment_name, + supports_rerunning_failed_realizations=supports_rerunning, + start_simulations_thread=start_fn, + cancel=client.stop, + has_failed_realizations=client.has_failed_realizations, ) self._dialog = RunDialog( f"Experiment - {self._config_file} {find_ert_info()}", - model.api, + run_model_api, event_queue, self._notifier, self.parent(), # type: ignore @@ -416,30 +428,35 @@ def run_experiment(self) -> None: storage_path=self._notifier.storage.path, ) self._dialog.queue_system.setText( - f"Queue system:\n{model.queue_config.queue_system.formatted_name}" + f"Queue system:\n{self.config.queue_config.queue_system.formatted_name}" ) self.experiment_started.emit(self._dialog) self._experiment_done = False self.run_button.setEnabled(self._experiment_done) - def start_simulation_thread(rerun_failed_realizations: bool = False) -> None: - simulation_thread = get_simulation_thread( - self._model, - rerun_failed_realizations, - use_ipc_protocol=self.config.queue_config.queue_system - == QueueSystem.LOCAL, + def do_rerun_failed() -> None: + try: + _new_run_id, _new_supports_rerunning = client.rerun_failed() + except Exception as e: + QMessageBox.warning( + self, + "ERROR: Failed to rerun", + str(e), + QMessageBox.StandardButton.Ok, + ) + return + _, rerun_thread = client.setup_event_queue_from_ws_endpoint( + event_queue=event_queue ) - self._dialog.setup_event_monitoring(rerun_failed_realizations) - simulation_thread.start() + self._dialog.setup_event_monitoring(rerun_failed_realizations=True) + rerun_thread.start() self._notifier.set_is_experiment_running(True) - def rerun_failed_realizations() -> None: - start_simulation_thread(rerun_failed_realizations=True) + self._dialog.rerun_failed_realizations_experiment.connect(do_rerun_failed) - self._dialog.rerun_failed_realizations_experiment.connect( - rerun_failed_realizations - ) - start_simulation_thread(rerun_failed_realizations=False) + self._dialog.setup_event_monitoring(rerun_failed_realizations=False) + monitor_thread.start() + self._notifier.set_is_experiment_running(True) def simulation_done_handler() -> None: self._experiment_done = True diff --git a/src/everest/gui/main_window.py b/src/everest/gui/main_window.py index a407ccc5afe..33a81c7c3bb 100644 --- a/src/everest/gui/main_window.py +++ b/src/everest/gui/main_window.py @@ -70,7 +70,10 @@ def run(self) -> None: title = Path(config["config_path"]).name self.setWindowTitle(f"EVEREST - {title}") - run_model_api = client.create_run_model_api() + run_model_api = client.create_run_model_api( + experiment_name=title, + supports_rerunning_failed_realizations=False, + ) event_queue, event_monitor_thread = client.setup_event_queue_from_ws_endpoint( refresh_interval=0.02, open_timeout=40, websocket_recv_timeout=1.0 ) diff --git a/src/everest/strings.py b/src/everest/strings.py index 5cacb641696..95df5ee370d 100644 --- a/src/everest/strings.py +++ b/src/everest/strings.py @@ -34,3 +34,7 @@ class EverEndpoints(StrEnum): runs = "runs" status = "status" events = "events" + check_runpath = "check_runpath" + delete_runpaths = "delete_runpaths" + rerun_failed = "rerun_failed" + has_failed_realizations = "has_failed_realizations" diff --git a/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py b/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py new file mode 100644 index 00000000000..a14e5d61bd8 --- /dev/null +++ b/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py @@ -0,0 +1,267 @@ +"""Unit tests for the new experiment_server endpoints: +- POST /check_runpath +- POST /delete_runpaths +- POST /rerun_failed/{run_id} +- GET /has_failed_realizations/{run_id} +""" + +from __future__ import annotations + +import shutil +from argparse import Namespace +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from ert.config import ErtConfig +from ert.dark_storage.app import app +from ert.dark_storage.endpoints.experiment_server import ( + ExperimentRunnerState, + _runs, +) +from ert.mode_definitions import ENSEMBLE_EXPERIMENT_MODE +from ert.run_models.model_factory import build_run_model_config +from everest.strings import EverEndpoints + +_TOKEN = "test-token" +_AUTH = ("__token__", _TOKEN) + +check_runpath_url = f"/experiment_server/{EverEndpoints.check_runpath}" +_DELETE_RUNPATHS_URL = f"/experiment_server/{EverEndpoints.delete_runpaths}" + + +@pytest.fixture +def experiment_server_client(monkeypatch): + """TestClient with ERT_STORAGE_TOKEN set.""" + original = dict(_runs) + monkeypatch.setenv("ERT_STORAGE_TOKEN", _TOKEN) + with TestClient(app) as client: + yield client + _runs.clear() + _runs.update(original) + + +@pytest.fixture +def poly_run_model_config(copy_poly_case): + """Returns a serialized EnsembleExperiment RunModelConfig for poly_example.""" + config = ErtConfig.from_file("poly.ert") + args = Namespace( + mode=ENSEMBLE_EXPERIMENT_MODE, + realizations="0,1", + experiment_name="preflight_test", + current_ensemble="default", + ) + return build_run_model_config(config, args).model_dump(mode="json") + + +@pytest.fixture +def copy_poly_case(tmp_path, source_root, monkeypatch): + poly_dir = tmp_path / "poly_example" + shutil.copytree( + source_root / "test-data" / "ert" / "poly_example", + poly_dir, + ignore=shutil.ignore_patterns("*ipynb", "poly_out", "storage", "logs"), + ) + monkeypatch.chdir(poly_dir) + return poly_dir + + +def test_that_check_runpath_returns_false_when_no_runpath_exists( + experiment_server_client, poly_run_model_config +): + response = experiment_server_client.post( + check_runpath_url, json=poly_run_model_config, auth=_AUTH + ) + assert response.status_code == 200 + data = response.json() + assert data["runpath_exists"] is False + assert data["num_existing"] == 0 + assert data["num_active"] == 2 + + +def test_that_check_runpath_returns_true_when_runpath_exists( + experiment_server_client, poly_run_model_config, tmp_path +): + # Create the runpath directories that the config expects + + runpath_format = poly_run_model_config["runpath_config"]["runpath_format_string"] + # Create realization 0, iter 0 directory + runpath_0 = ( + Path(runpath_format % (0, 0)) + if "%d" in runpath_format + else Path(runpath_format.replace("", "0").replace("", "0")) + ) + runpath_0.mkdir(parents=True, exist_ok=True) + + response = experiment_server_client.post( + check_runpath_url, json=poly_run_model_config, auth=_AUTH + ) + assert response.status_code == 200 + data = response.json() + assert data["runpath_exists"] is True + + +def test_that_check_runpath_requires_authentication( + experiment_server_client, poly_run_model_config +): + response = experiment_server_client.post( + check_runpath_url, json=poly_run_model_config + ) + assert response.status_code == 401 + + +def test_that_check_runpath_returns_422_for_invalid_config( + experiment_server_client, +): + response = experiment_server_client.post( + check_runpath_url, json={"type": "invalid_type"}, auth=_AUTH + ) + assert response.status_code == 422 + + +def test_that_delete_runpaths_removes_existing_run_directories( + experiment_server_client, poly_run_model_config, tmp_path +): + runpath_format = poly_run_model_config["runpath_config"]["runpath_format_string"] + # Substitute template placeholders — use ERT's realization/iter format + runpath_0 = Path( + runpath_format.replace("", "0").replace("", "0") + if "" in runpath_format + else runpath_format % (0, 0) + ) + runpath_0.mkdir(parents=True, exist_ok=True) + assert runpath_0.exists() + + response = experiment_server_client.post( + _DELETE_RUNPATHS_URL, json=poly_run_model_config, auth=_AUTH + ) + assert response.status_code == 200 + assert not runpath_0.exists() + + +def test_that_delete_runpaths_requires_authentication( + experiment_server_client, poly_run_model_config +): + response = experiment_server_client.post( + _DELETE_RUNPATHS_URL, json=poly_run_model_config + ) + assert response.status_code == 401 + + +def test_that_has_failed_realizations_returns_false_when_no_realizations_failed( + experiment_server_client, +): + run_id = "test-run-no-failed" + state = ExperimentRunnerState(has_failed_realizations=False) + _runs[run_id] = state + + response = experiment_server_client.get( + f"/experiment_server/{EverEndpoints.has_failed_realizations}/{run_id}", + auth=_AUTH, + ) + assert response.status_code == 200 + assert response.json() == {"has_failed": False} + + +def test_that_has_failed_realizations_returns_true_when_some_realizations_failed( + experiment_server_client, +): + run_id = "test-run-with-failed" + state = ExperimentRunnerState(has_failed_realizations=True) + _runs[run_id] = state + + response = experiment_server_client.get( + f"/experiment_server/{EverEndpoints.has_failed_realizations}/{run_id}", + auth=_AUTH, + ) + assert response.status_code == 200 + assert response.json() == {"has_failed": True} + + +def test_that_has_failed_realizations_returns_404_for_unknown_run_id( + experiment_server_client, +): + response = experiment_server_client.get( + f"/experiment_server/{EverEndpoints.has_failed_realizations}/nonexistent-id", + auth=_AUTH, + ) + assert response.status_code == 404 + + +def test_that_rerun_failed_creates_new_run_with_same_model( + experiment_server_client, +): + mock_run_model = MagicMock() + mock_run_model.supports_rerunning_failed_realizations = True + + run_id = "original-run-id" + state = ExperimentRunnerState( + run_model=mock_run_model, + supports_rerunning_failed_realizations=True, + config_path="/path/to/config.ert", + run_path="/path/to/runpath", + storage_path="/path/to/storage", + ) + _runs[run_id] = state + + with patch( + "ert.dark_storage.endpoints.experiment_server.ExperimentRunner.run" + ) as mock_run: + mock_run.return_value = None + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.rerun_failed}/{run_id}", + auth=_AUTH, + ) + + assert response.status_code == 200 + data = response.json() + new_run_id = data["new_run_id"] + assert new_run_id != run_id + assert new_run_id in _runs + assert _runs[new_run_id].run_model is mock_run_model + assert data["supports_rerunning_failed_realizations"] is True + + +def test_that_rerun_failed_returns_400_when_run_model_is_missing( + experiment_server_client, +): + run_id = "run-without-model" + state = ExperimentRunnerState(run_model=None) + _runs[run_id] = state + + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.rerun_failed}/{run_id}", + auth=_AUTH, + ) + assert response.status_code == 400 + + +def test_that_rerun_failed_returns_400_when_rerun_is_not_supported( + experiment_server_client, +): + mock_run_model = MagicMock() + + run_id = "run-no-rerun-support" + state = ExperimentRunnerState( + run_model=mock_run_model, + supports_rerunning_failed_realizations=False, + ) + _runs[run_id] = state + + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.rerun_failed}/{run_id}", + auth=_AUTH, + ) + assert response.status_code == 400 + + +def test_that_rerun_failed_returns_404_for_unknown_run_id( + experiment_server_client, +): + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.rerun_failed}/nonexistent-id", + auth=_AUTH, + ) + assert response.status_code == 404 From 167adc5dcf8be962d9799768ce12540c237542cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Wed, 6 May 2026 22:29:57 +0200 Subject: [PATCH 06/11] Rewrite restart --- .../endpoints/experiment_server.py | 78 +++++++++---------- src/ert/gui/experiments/experiment_client.py | 6 +- src/everest/strings.py | 1 - .../test_experiment_server_new_endpoints.py | 16 ++-- 4 files changed, 51 insertions(+), 50 deletions(-) diff --git a/src/ert/dark_storage/endpoints/experiment_server.py b/src/ert/dark_storage/endpoints/experiment_server.py index 3045ca68eee..69e761c630c 100644 --- a/src/ert/dark_storage/endpoints/experiment_server.py +++ b/src/ert/dark_storage/endpoints/experiment_server.py @@ -220,10 +220,47 @@ async def start_experiment( request: Request, background_tasks: BackgroundTasks, credentials: Annotated[HTTPBasicCredentials, Depends(security)], + rerun_from_run_id: str | None = None, ) -> JSONResponse: _log(request) _check_user(credentials) run_id = str(uuid.uuid4()) + if rerun_from_run_id is not None: + if rerun_from_run_id not in _runs: + raise HTTPException( + status_code=404, detail=f"Run '{rerun_from_run_id}' not found" + ) + source_run = _runs[rerun_from_run_id] + if source_run.run_model is None: + raise HTTPException( + status_code=400, + detail=f"Run '{rerun_from_run_id}' has no run model to rerun.", + ) + if not source_run.supports_rerunning_failed_realizations: + raise HTTPException( + status_code=400, + detail=f"Run '{rerun_from_run_id}' " + f"does not support rerunning failed realizations.", + ) + run_state = ExperimentRunnerState( + config_path=source_run.config_path, + run_path=source_run.run_path, + storage_path=source_run.storage_path, + run_model=source_run.run_model, + supports_rerunning_failed_realizations=source_run.supports_rerunning_failed_realizations, + ) + _runs[run_id] = run_state + runner = ExperimentRunner(None, run_id) + background_tasks.add_task(runner.run, rerun=True) + run_state.start_time_unix = int(time.time()) + return JSONResponse( + { + "run_id": run_id, + "supports_rerunning_failed_realizations": ( + run_state.supports_rerunning_failed_realizations + ), + } + ) run_state = ExperimentRunnerState() _runs[run_id] = run_state request_data = await request.json() @@ -357,47 +394,6 @@ async def delete_runpaths( ) from e -@router.post(f"/{EverEndpoints.rerun_failed}/{{run_id}}") -async def rerun_failed( - request: Request, - run: Annotated[ExperimentRunnerState, Depends(_get_run)], - run_id: str, - background_tasks: BackgroundTasks, - credentials: Annotated[HTTPBasicCredentials, Depends(security)], -) -> JSONResponse: - _log(request) - _check_user(credentials) - if run.run_model is None: - raise HTTPException( - status_code=400, detail=f"Run '{run_id}' has no run model to rerun." - ) - if not run.supports_rerunning_failed_realizations: - raise HTTPException( - status_code=400, - detail=f"Run '{run_id}' does not support rerunning failed realizations.", - ) - new_run_id = str(uuid.uuid4()) - new_run_state = ExperimentRunnerState( - config_path=run.config_path, - run_path=run.run_path, - storage_path=run.storage_path, - run_model=run.run_model, - supports_rerunning_failed_realizations=run.supports_rerunning_failed_realizations, - ) - _runs[new_run_id] = new_run_state - runner = ExperimentRunner(None, new_run_id) - background_tasks.add_task(runner.run, rerun=True) - new_run_state.start_time_unix = int(time.time()) - return JSONResponse( - { - "new_run_id": new_run_id, - "supports_rerunning_failed_realizations": ( - new_run_state.supports_rerunning_failed_realizations - ), - } - ) - - @router.get(f"/{EverEndpoints.has_failed_realizations}/{{run_id}}") def has_failed_realizations_endpoint( request: Request, diff --git a/src/ert/gui/experiments/experiment_client.py b/src/ert/gui/experiments/experiment_client.py index a92ddd38372..e995b7411fd 100644 --- a/src/ert/gui/experiments/experiment_client.py +++ b/src/ert/gui/experiments/experiment_client.py @@ -95,10 +95,12 @@ def start_experiment(self, config_json: dict[str, Any]) -> tuple[str, bool]: def rerun_failed(self) -> tuple[str, bool]: assert self._run_id is not None, "No active run to rerun" - response = self._http_post(f"{EverEndpoints.rerun_failed}/{self._run_id}") + response = self._http_post( + f"{EverEndpoints.start_experiment}?rerun_from_run_id={self._run_id}" + ) response.raise_for_status() data = response.json() - self._run_id = data["new_run_id"] + self._run_id = data["run_id"] return self._run_id, data.get("supports_rerunning_failed_realizations", False) def has_failed_realizations(self) -> bool: diff --git a/src/everest/strings.py b/src/everest/strings.py index 95df5ee370d..4d81add53d1 100644 --- a/src/everest/strings.py +++ b/src/everest/strings.py @@ -36,5 +36,4 @@ class EverEndpoints(StrEnum): events = "events" check_runpath = "check_runpath" delete_runpaths = "delete_runpaths" - rerun_failed = "rerun_failed" has_failed_realizations = "has_failed_realizations" diff --git a/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py b/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py index a14e5d61bd8..05ea8506623 100644 --- a/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py +++ b/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py @@ -1,7 +1,7 @@ """Unit tests for the new experiment_server endpoints: - POST /check_runpath - POST /delete_runpaths -- POST /rerun_failed/{run_id} +- POST /start_experiment?rerun_from_run_id={run_id} - GET /has_failed_realizations/{run_id} """ @@ -211,13 +211,14 @@ def test_that_rerun_failed_creates_new_run_with_same_model( ) as mock_run: mock_run.return_value = None response = experiment_server_client.post( - f"/experiment_server/{EverEndpoints.rerun_failed}/{run_id}", + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": run_id}, auth=_AUTH, ) assert response.status_code == 200 data = response.json() - new_run_id = data["new_run_id"] + new_run_id = data["run_id"] assert new_run_id != run_id assert new_run_id in _runs assert _runs[new_run_id].run_model is mock_run_model @@ -232,7 +233,8 @@ def test_that_rerun_failed_returns_400_when_run_model_is_missing( _runs[run_id] = state response = experiment_server_client.post( - f"/experiment_server/{EverEndpoints.rerun_failed}/{run_id}", + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": run_id}, auth=_AUTH, ) assert response.status_code == 400 @@ -251,7 +253,8 @@ def test_that_rerun_failed_returns_400_when_rerun_is_not_supported( _runs[run_id] = state response = experiment_server_client.post( - f"/experiment_server/{EverEndpoints.rerun_failed}/{run_id}", + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": run_id}, auth=_AUTH, ) assert response.status_code == 400 @@ -261,7 +264,8 @@ def test_that_rerun_failed_returns_404_for_unknown_run_id( experiment_server_client, ): response = experiment_server_client.post( - f"/experiment_server/{EverEndpoints.rerun_failed}/nonexistent-id", + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": "nonexistent-id"}, auth=_AUTH, ) assert response.status_code == 404 From 88ff9cb5b8d1a71f234a0275ba74e05af8c0dc44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Thu, 7 May 2026 09:44:15 +0200 Subject: [PATCH 07/11] Fix test --- .../gui/experiments/test_run_dialog.py | 62 ++++++++++++------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py index a827de38894..32e03424dcf 100644 --- a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py +++ b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py @@ -1,6 +1,7 @@ import tempfile from pathlib import Path from queue import SimpleQueue +from threading import Thread from unittest.mock import MagicMock, Mock, patch import pandas as pd @@ -58,20 +59,6 @@ from tests.ert.ui_tests.gui.conftest import wait_for_child -@pytest.fixture -def event_queue(events): - async def _add_event(self, *_): - for event in events: - self.send_event(event) - return [0] - - with patch( - "ert.run_models.run_model.RunModel.run_ensemble_evaluator_async", - _add_event, - ): - yield - - @pytest.fixture def event_queue_large_snapshot(large_snapshot): events = [ @@ -125,7 +112,7 @@ def mock_set_env_key(): @pytest.fixture -def run_dialog(qtbot: QtBot, use_tmpdir, mock_set_env_key, monkeypatch): +def run_dialog(events, qtbot: QtBot, use_tmpdir, mock_set_env_key, monkeypatch): config_file = "minimal_config.ert" monkeypatch.setattr("ert.scheduler.Scheduler.BATCH_KILLING_INTERVAL", 0.01) monkeypatch.setattr( @@ -148,8 +135,37 @@ def run_dialog(qtbot: QtBot, use_tmpdir, mock_set_env_key, monkeypatch): simulation_settings._experiment_name_field.setText("new_experiment_name") run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") assert run_experiment - qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) - qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) + + def _make_mock_client(): + mock_client = MagicMock() + mock_client.check_runpath.return_value = { + "runpath_exists": False, + "num_existing": 0, + "num_active": 1, + } + mock_client.start_experiment.return_value = ("mock-run-id", False) + + def _setup_event_queue(event_queue=None, **_kwargs): + if event_queue is None: + event_queue = SimpleQueue() + + def _feed_events(): + for event in events: + event_queue.put(event) + + thread = Thread(target=_feed_events, daemon=True) + return event_queue, thread + + mock_client.setup_event_queue_from_ws_endpoint.side_effect = _setup_event_queue + return mock_client + + with patch.object( + ExperimentPanel, + "_create_experiment_client", + side_effect=_make_mock_client, + ): + qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) + qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) run_dialog = gui.findChild(RunDialog) assert run_dialog return run_dialog @@ -440,7 +456,7 @@ def test_large_snapshot( ), ], ) -def test_run_dialog(events, event_queue, tab_widget_count, qtbot: QtBot, run_dialog): +def test_run_dialog(events, tab_widget_count, qtbot: QtBot, run_dialog): qtbot.waitUntil( lambda: run_dialog._tab_widget.count() == tab_widget_count, timeout=5000 ) @@ -514,7 +530,7 @@ def test_run_dialog(events, event_queue, tab_widget_count, qtbot: QtBot, run_dia ], ) def test_run_dialog_memory_usage_showing( - events, event_queue, tab_widget_count, qtbot: QtBot, run_dialog + events, tab_widget_count, qtbot: QtBot, run_dialog ): qtbot.waitUntil( lambda: run_dialog._tab_widget.count() == tab_widget_count, timeout=5000 @@ -615,7 +631,7 @@ def test_run_dialog_memory_usage_showing( ], ) def test_run_dialog_fm_label_show_correct_info( - events, event_queue, tab_widget_count, expected_host_info, qtbot: QtBot, run_dialog + events, tab_widget_count, expected_host_info, qtbot: QtBot, run_dialog ): qtbot.waitUntil( lambda: run_dialog._tab_widget.count() == tab_widget_count, timeout=5000 @@ -891,7 +907,7 @@ def dialog_appeared_and_test(): ], ) def test_forward_model_overview_label_selected_on_tab_change( - events, event_queue, tab_widget_count, qtbot: QtBot, run_dialog + events, tab_widget_count, qtbot: QtBot, run_dialog ): def qt_bot_click_realization(realization_index: int, iteration: int) -> None: view = run_dialog._tab_widget.widget(iteration)._real_view @@ -1033,7 +1049,7 @@ def test_that_file_dialog_close_when_run_dialog_hidden(qtbot: QtBot, run_dialog) ], ) def test_that_run_dialog_clears_warnings_when_rerun( - events, event_queue, qtbot, monkeypatch, run_dialog + events, qtbot, monkeypatch, run_dialog ): qtbot.wait_until(run_dialog.is_experiment_done, timeout=5000) assert len(run_dialog.post_experiment_warnings) > 0 @@ -1098,7 +1114,7 @@ def test_that_run_dialog_clears_warnings_when_rerun( ], ) def test_that_experiment_with_a_scheduler_warning_event_shows_a_warning_dialog( - events, event_queue, qtbot: QtBot, run_dialog: RunDialog + events, qtbot: QtBot, run_dialog: RunDialog ): ensemble_evaluation_warning_box = wait_for_child(run_dialog, qtbot, QMessageBox) From a8a54facefaeb4c71a0a6353e0b97f09348e71cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Fri, 8 May 2026 10:30:53 +0200 Subject: [PATCH 08/11] Try to fix it --- .../endpoints/experiment_server.py | 1 - .../gui/experiments/test_run_dialog.py | 72 ++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/ert/dark_storage/endpoints/experiment_server.py b/src/ert/dark_storage/endpoints/experiment_server.py index 69e761c630c..578cbdf8007 100644 --- a/src/ert/dark_storage/endpoints/experiment_server.py +++ b/src/ert/dark_storage/endpoints/experiment_server.py @@ -467,7 +467,6 @@ async def run(self, rerun: bool = False) -> None: try: if rerun and run.run_model is not None: run_model = run.run_model - # Rewire status_queue so events go to this run's event loop run_model._status_queue = status_queue else: assert self._config is not None, ( diff --git a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py index 32e03424dcf..baaaf4b6fa7 100644 --- a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py +++ b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py @@ -1,7 +1,10 @@ +import json import tempfile +from base64 import b64encode from pathlib import Path from queue import SimpleQueue from threading import Thread +from typing import Any from unittest.mock import MagicMock, Mock, patch import pandas as pd @@ -18,10 +21,15 @@ QWidget, ) from pytestqt.qtbot import QtBot +from starlette.testclient import TestClient import ert.run_models from _ert.events import EnsembleEvaluationWarning from ert.config import ErtConfig +from ert.dark_storage.app import app +from ert.dark_storage.endpoints.experiment_server import ( + _runs, +) from ert.ensemble_evaluator import state from ert.ensemble_evaluator.event import ( EndEvent, @@ -36,6 +44,7 @@ EnsembleExperimentPanel, ) from ert.gui.experiments.ensemble_smoother_panel import EnsembleSmootherPanel +from ert.gui.experiments.experiment_client import ExperimentClient from ert.gui.experiments.multiple_data_assimilation_panel import ( MultipleDataAssimilationPanel, ) @@ -59,6 +68,67 @@ from tests.ert.ui_tests.gui.conftest import wait_for_child +@pytest.fixture +def start_experiment_client(monkeypatch): + original = dict(_runs) + # with ErtServerController.init_service( + # timeout=240, project=tmp_path_factory.mktemp("server_path") + # ) as server: + # with ErtServerController.init_service(project=Path().absolute()) as server: + # server.fetch_connection_info() + # yield + monkeypatch.setenv("ERT_STORAGE_TOKEN", "password") + client = TestClient(app) + credentials = b64encode(b"username:password").decode() + + class ExperimentClientMock(ExperimentClient): + def _http_post_json(self, endpoint: str, body: dict[str, Any]): + return client.post( + f"{self._url}/{endpoint}", + json=body, + headers={"Authorization": f"Basic {credentials}"}, + ) + + def check_runpath(self, config_json: dict[str, Any]) -> dict[str, Any]: + return { + "runpath_exists": False, + "num_existing": 0, + "num_active": 1, + } + + client_mock = ExperimentClientMock( + url="/experiment_server", + cert_file="password", + username="username", + password="password", + ssl_context=None, + ) # type: ignore + # client_mock.check_runpath = MagicMock(return_value={ + # "runpath_exists": False, + # "num_existing": 0, + # "num_active": 1, + # }) + + monkeypatch.setattr( + ExperimentPanel, "_create_experiment_client", lambda self: client_mock + ) + + credentials = b64encode(b"username:password").decode() + storage_path = Path() / "storage" + storage_path.mkdir(exist_ok=True) + mock_server_json = { + "urls": "http://testserver", + "cert": "password", + "authtoken": "__token__", + } + (storage_path / "storage_server.json").write_text( + json.dumps(mock_server_json), encoding="utf-8" + ) + yield client, {"Authorization": f"Basic {credentials}"} + _runs.clear() + _runs.update(original) + + @pytest.fixture def event_queue_large_snapshot(large_snapshot): events = [ @@ -175,7 +245,7 @@ def _feed_events(): @pytest.mark.timeout(10) @pytest.mark.slow def test_that_terminating_experiment_shows_a_confirmation_dialog( - qtbot: QtBot, monkeypatch + qtbot: QtBot, monkeypatch, start_experiment_client ): config_file = "minimal_config.ert" monkeypatch.setattr("ert.scheduler.Scheduler.BATCH_KILLING_INTERVAL", 0.01) From 6ad89bd0177960e7c401d3b11f4435ec1721e616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Fri, 8 May 2026 12:41:30 +0200 Subject: [PATCH 09/11] Try to fix it 2 --- .../endpoints/experiment_server.py | 35 +-- .../gui/experiments/test_run_dialog.py | 249 +++++++----------- 2 files changed, 113 insertions(+), 171 deletions(-) diff --git a/src/ert/dark_storage/endpoints/experiment_server.py b/src/ert/dark_storage/endpoints/experiment_server.py index 578cbdf8007..e7446ae3021 100644 --- a/src/ert/dark_storage/endpoints/experiment_server.py +++ b/src/ert/dark_storage/endpoints/experiment_server.py @@ -489,10 +489,11 @@ async def run(self, rerun: bool = False) -> None: else EvaluatorServerConfig(use_ipc_protocol=False) ), ) + cancelled = False while True: - if run.status.status == ExperimentState.stopped: + if run.status.status == ExperimentState.stopped and not cancelled: run_model.cancel() - raise UserCancelled("Optimization aborted") + cancelled = True try: item: StatusEvents = status_queue.get(block=False) except queue.Empty: @@ -509,20 +510,22 @@ async def run(self, rerun: bool = False) -> None: await sub.is_done() break await simulation_future - if isinstance(run_model, EverestRunModel): - assert run_model.exit_code is not None - exp_state, msg = _get_optimization_status( - run_model.exit_code, run.events - ) - run_status = ExperimentStatus( - message=msg, - status=exp_state, - ) - else: - run_status = ExperimentStatus( - message="Experiment completed.", status=ExperimentState.completed - ) - run.status = run_status + if not cancelled: + if isinstance(run_model, EverestRunModel): + assert run_model.exit_code is not None + exp_state, msg = _get_optimization_status( + run_model.exit_code, run.events + ) + run_status = ExperimentStatus( + message=msg, + status=exp_state, + ) + else: + run_status = ExperimentStatus( + message="Experiment completed.", + status=ExperimentState.completed, + ) + run.status = run_status except UserCancelled as e: logging.getLogger(EXPERIMENT_SERVER).info(f"User cancelled: {e}") except Exception as e: diff --git a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py index baaaf4b6fa7..c74e8a143a9 100644 --- a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py +++ b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py @@ -1,10 +1,7 @@ -import json import tempfile -from base64 import b64encode from pathlib import Path from queue import SimpleQueue from threading import Thread -from typing import Any from unittest.mock import MagicMock, Mock, patch import pandas as pd @@ -21,15 +18,10 @@ QWidget, ) from pytestqt.qtbot import QtBot -from starlette.testclient import TestClient import ert.run_models from _ert.events import EnsembleEvaluationWarning from ert.config import ErtConfig -from ert.dark_storage.app import app -from ert.dark_storage.endpoints.experiment_server import ( - _runs, -) from ert.ensemble_evaluator import state from ert.ensemble_evaluator.event import ( EndEvent, @@ -44,7 +36,6 @@ EnsembleExperimentPanel, ) from ert.gui.experiments.ensemble_smoother_panel import EnsembleSmootherPanel -from ert.gui.experiments.experiment_client import ExperimentClient from ert.gui.experiments.multiple_data_assimilation_panel import ( MultipleDataAssimilationPanel, ) @@ -63,72 +54,12 @@ StartingTotalRunPathCreationEvent, ) from ert.scheduler.job import Job +from ert.services import ErtServerController from tests.ert import SnapshotBuilder from tests.ert.handle_run_path_dialog import handle_run_path_dialog from tests.ert.ui_tests.gui.conftest import wait_for_child -@pytest.fixture -def start_experiment_client(monkeypatch): - original = dict(_runs) - # with ErtServerController.init_service( - # timeout=240, project=tmp_path_factory.mktemp("server_path") - # ) as server: - # with ErtServerController.init_service(project=Path().absolute()) as server: - # server.fetch_connection_info() - # yield - monkeypatch.setenv("ERT_STORAGE_TOKEN", "password") - client = TestClient(app) - credentials = b64encode(b"username:password").decode() - - class ExperimentClientMock(ExperimentClient): - def _http_post_json(self, endpoint: str, body: dict[str, Any]): - return client.post( - f"{self._url}/{endpoint}", - json=body, - headers={"Authorization": f"Basic {credentials}"}, - ) - - def check_runpath(self, config_json: dict[str, Any]) -> dict[str, Any]: - return { - "runpath_exists": False, - "num_existing": 0, - "num_active": 1, - } - - client_mock = ExperimentClientMock( - url="/experiment_server", - cert_file="password", - username="username", - password="password", - ssl_context=None, - ) # type: ignore - # client_mock.check_runpath = MagicMock(return_value={ - # "runpath_exists": False, - # "num_existing": 0, - # "num_active": 1, - # }) - - monkeypatch.setattr( - ExperimentPanel, "_create_experiment_client", lambda self: client_mock - ) - - credentials = b64encode(b"username:password").decode() - storage_path = Path() / "storage" - storage_path.mkdir(exist_ok=True) - mock_server_json = { - "urls": "http://testserver", - "cert": "password", - "authtoken": "__token__", - } - (storage_path / "storage_server.json").write_text( - json.dumps(mock_server_json), encoding="utf-8" - ) - yield client, {"Authorization": f"Basic {credentials}"} - _runs.clear() - _runs.update(original) - - @pytest.fixture def event_queue_large_snapshot(large_snapshot): events = [ @@ -242,10 +173,10 @@ def _feed_events(): @pytest.mark.usefixtures("use_tmpdir") -@pytest.mark.timeout(10) +@pytest.mark.timeout(60) @pytest.mark.slow def test_that_terminating_experiment_shows_a_confirmation_dialog( - qtbot: QtBot, monkeypatch, start_experiment_client + qtbot: QtBot, monkeypatch ): config_file = "minimal_config.ert" monkeypatch.setattr("ert.scheduler.Scheduler.BATCH_KILLING_INTERVAL", 0.01) @@ -278,48 +209,51 @@ def test_that_terminating_experiment_shows_a_confirmation_dialog( ) args_mock = Mock() args_mock.config = config_file - gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") - qtbot.addWidget(gui) - experiment_panel = gui.findChild(ExperimentPanel) - assert experiment_panel - simulation_mode_combo = experiment_panel.findChild(QComboBox) - assert simulation_mode_combo - simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) - simulation_settings = gui.findChild(EnsembleExperimentPanel) - simulation_settings._experiment_name_field.setText("new_experiment_name") - run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") - assert run_experiment - qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) - qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) - run_dialog = gui.findChild(RunDialog) - assert run_dialog - kill_button = run_dialog.kill_button - with qtbot.waitSignal(run_dialog.experiment_done, timeout=10000): - # Wait for ensemble to start evaluating before cancelling - _ = wait_for_child(run_dialog, qtbot, RealizationWidget) - - def handle_dialog(): - terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) - dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() - yes_button = next(b for b in dialog_buttons if "Yes" in b.text()) - qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) - - QTimer.singleShot(100, handle_dialog) - assert kill_button.isEnabled() + storage_path = Path("storage") + storage_path.mkdir() + with ErtServerController.init_service(project=storage_path.absolute()): + gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") + qtbot.addWidget(gui) + experiment_panel = gui.findChild(ExperimentPanel) + assert experiment_panel + simulation_mode_combo = experiment_panel.findChild(QComboBox) + assert simulation_mode_combo + simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) + simulation_settings = gui.findChild(EnsembleExperimentPanel) + simulation_settings._experiment_name_field.setText("new_experiment_name") + run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") + assert run_experiment + qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) + qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=30000) + run_dialog = gui.findChild(RunDialog) + assert run_dialog + kill_button = run_dialog.kill_button + with qtbot.waitSignal(run_dialog.experiment_done, timeout=30000): + # Wait for ensemble to start evaluating before cancelling + _ = wait_for_child(run_dialog, qtbot, RealizationWidget) + + def handle_dialog(): + terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) + dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() + yes_button = next(b for b in dialog_buttons if "Yes" in b.text()) + qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) + + QTimer.singleShot(100, handle_dialog) + assert kill_button.isEnabled() + assert kill_button.text() == "Terminate experiment" + qtbot.mouseClick(run_dialog.kill_button, Qt.MouseButton.LeftButton) + assert not kill_button.isEnabled() + assert kill_button.text() == "Terminating" + wait_for_child(run_dialog, qtbot, Suggestor) + assert run_dialog.fail_msg_box is not None assert kill_button.text() == "Terminate experiment" - qtbot.mouseClick(run_dialog.kill_button, Qt.MouseButton.LeftButton) assert not kill_button.isEnabled() - assert kill_button.text() == "Terminating" - wait_for_child(run_dialog, qtbot, Suggestor) - assert run_dialog.fail_msg_box is not None - assert kill_button.text() == "Terminate experiment" - assert not kill_button.isEnabled() - assert ( - "Experiment cancelled by user during evaluation" - in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") - .findChild(QLabel) - .text() - ) + assert ( + "Experiment cancelled by user during evaluation" + in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") + .findChild(QLabel) + .text() + ) @pytest.mark.slow @@ -1268,7 +1202,7 @@ def test_that_runpath_creation_events_add_update_and_remove_tab(qtbot: QtBot) -> @pytest.mark.usefixtures("use_tmpdir") -@pytest.mark.timeout(20) +@pytest.mark.timeout(60) @pytest.mark.slow def test_that_terminating_experiment_during_hooked_workflows_stops_run_dialog( qtbot: QtBot, monkeypatch @@ -1341,50 +1275,55 @@ def test_that_terminating_experiment_during_hooked_workflows_stops_run_dialog( args_mock = Mock() args_mock.config = config_file ert_config = ErtConfig.from_file(config_file) - gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") - qtbot.addWidget(gui) - experiment_panel = gui.findChild(ExperimentPanel) - assert experiment_panel - simulation_mode_combo = experiment_panel.findChild(QComboBox) - assert simulation_mode_combo - simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) - simulation_settings = gui.findChild(EnsembleExperimentPanel) - simulation_settings._experiment_name_field.setText("new_experiment_name") - run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") - assert run_experiment - qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) - qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) - run_dialog = gui.findChild(RunDialog) - assert run_dialog - kill_button = run_dialog.kill_button + storage_path = Path("storage") + storage_path.mkdir() + with ErtServerController.init_service(project=storage_path.absolute()): + gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") + qtbot.addWidget(gui) + experiment_panel = gui.findChild(ExperimentPanel) + assert experiment_panel + simulation_mode_combo = experiment_panel.findChild(QComboBox) + assert simulation_mode_combo + simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) + simulation_settings = gui.findChild(EnsembleExperimentPanel) + simulation_settings._experiment_name_field.setText("new_experiment_name") + run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") + assert run_experiment + qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) + qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=30000) + run_dialog = gui.findChild(RunDialog) + assert run_dialog + kill_button = run_dialog.kill_button - qtbot.waitUntil(file_a.exists, timeout=10000) + qtbot.waitUntil(file_a.exists, timeout=30000) - def handle_dialog() -> None: - terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) - dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() - yes_button = next(button for button in dialog_buttons if "Yes" in button.text()) - qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) + def handle_dialog() -> None: + terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) + dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() + yes_button = next( + button for button in dialog_buttons if "Yes" in button.text() + ) + qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) - with qtbot.waitSignal(run_dialog.experiment_done, timeout=10000): - QTimer.singleShot(100, handle_dialog) - assert kill_button.isEnabled() - assert kill_button.text() == "Terminate experiment" - qtbot.mouseClick(kill_button, Qt.MouseButton.LeftButton) - assert not kill_button.isEnabled() - assert kill_button.text() == "Terminating" + with qtbot.waitSignal(run_dialog.experiment_done, timeout=30000): + QTimer.singleShot(100, handle_dialog) + assert kill_button.isEnabled() + assert kill_button.text() == "Terminate experiment" + qtbot.mouseClick(kill_button, Qt.MouseButton.LeftButton) + assert not kill_button.isEnabled() + assert kill_button.text() == "Terminating" - qtbot.waitUntil(lambda: run_dialog.fail_msg_box is not None, timeout=5000) + qtbot.waitUntil(lambda: run_dialog.fail_msg_box is not None, timeout=5000) - assert file_a.exists() - assert not file_b.exists() - assert not file_c.exists() - assert run_dialog.fail_msg_box is not None - assert kill_button.text() == "Terminate experiment" - assert not kill_button.isEnabled() - assert ( - "Experiment cancelled by user during workflows" - in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") - .findChild(QLabel) - .text() - ) + assert file_a.exists() + assert not file_b.exists() + assert not file_c.exists() + assert run_dialog.fail_msg_box is not None + assert kill_button.text() == "Terminate experiment" + assert not kill_button.isEnabled() + assert ( + "Experiment cancelled by user during workflows" + in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") + .findChild(QLabel) + .text() + ) From 5a2163c536bec0125de6bfc818496101d045f88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Fri, 8 May 2026 15:32:33 +0200 Subject: [PATCH 10/11] fix? --- src/ert/dark_storage/endpoints/experiment_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ert/dark_storage/endpoints/experiment_server.py b/src/ert/dark_storage/endpoints/experiment_server.py index e7446ae3021..87f529a4c53 100644 --- a/src/ert/dark_storage/endpoints/experiment_server.py +++ b/src/ert/dark_storage/endpoints/experiment_server.py @@ -509,7 +509,11 @@ async def run(self, rerun: bool = False) -> None: for sub in list(run.subscribers.values()): await sub.is_done() break - await simulation_future + try: + await simulation_future + except Exception: + if not cancelled: + raise if not cancelled: if isinstance(run_model, EverestRunModel): assert run_model.exit_code is not None From e1f2d460dd7f73072ff3da0b2d6ecb64ff723c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Eide?= Date: Mon, 11 May 2026 10:52:18 +0200 Subject: [PATCH 11/11] fix? --- src/ert/logging/storage_log.conf | 2 ++ tests/ert/ui_tests/gui/conftest.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ert/logging/storage_log.conf b/src/ert/logging/storage_log.conf index b329bb53acf..d7f1fd74380 100644 --- a/src/ert/logging/storage_log.conf +++ b/src/ert/logging/storage_log.conf @@ -40,6 +40,8 @@ loggers: propagate: False ert.shared.status: level: INFO + ert.scheduler: + level: ERROR res: level: INFO res.config: diff --git a/tests/ert/ui_tests/gui/conftest.py b/tests/ert/ui_tests/gui/conftest.py index cc4990ec985..652b8586177 100644 --- a/tests/ert/ui_tests/gui/conftest.py +++ b/tests/ert/ui_tests/gui/conftest.py @@ -35,7 +35,7 @@ from ert.gui.tools.manage_experiments.storage_widget import AddWidget, StorageWidget from ert.plugins import get_site_plugins from ert.run_models import EnsembleExperiment, MultipleDataAssimilation -from ert.storage import Storage +from ert.services import ErtServerController from tests.ert.handle_run_path_dialog import handle_run_path_dialog DEFAULT_NUM_REALIZATIONS = 10 @@ -94,15 +94,19 @@ def _new_poly_example( @contextmanager -def _open_main_window(path) -> Iterator[tuple[ErtMainWindow, Storage, ErtConfig]]: +def _open_main_window(path) -> Iterator[tuple[ErtMainWindow, str, ErtConfig]]: args_mock = Mock() args_mock.config = str(path) site_plugins = get_site_plugins() with use_runtime_plugins(site_plugins): config = ErtConfig.with_plugins(site_plugins).from_file(path) + ens_path = Path(config.ens_path).absolute() + ens_path.mkdir(parents=True, exist_ok=True) with ( add_gui_log_handler() as log_handler, + ErtServerController.init_service(project=ens_path) as server, ): + server.wait_until_ready() gui = _setup_main_window(config, args_mock, log_handler, config.ens_path) yield gui, config.ens_path, config gui.close()