diff --git a/src/ert/dark_storage/endpoints/experiment_server.py b/src/ert/dark_storage/endpoints/experiment_server.py index 03a2698d240..87f529a4c53 100644 --- a/src/ert/dark_storage/endpoints/experiment_server.py +++ b/src/ert/dark_storage/endpoints/experiment_server.py @@ -1,6 +1,5 @@ import asyncio import dataclasses -import datetime import logging import os import queue @@ -11,7 +10,7 @@ import warnings from base64 import b64decode from queue import SimpleQueue -from typing import Annotated +from typing import TYPE_CHECKING, Annotated from fastapi import ( APIRouter, @@ -23,19 +22,23 @@ from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse from fastapi.security import HTTPBasic, HTTPBasicCredentials +from pydantic import TypeAdapter from starlette import status from starlette.requests import Request from starlette.responses import PlainTextResponse, Response from starlette.websockets import WebSocket -from ert.base_model_context import use_runtime_plugins from ert.config import ConfigWarning, QueueSystem from ert.ensemble_evaluator import EndEvent, EvaluatorServerConfig from ert.ensemble_evaluator.event import FullSnapshotEvent, SnapshotUpdateEvent from ert.ensemble_evaluator.snapshot import EnsembleSnapshot -from ert.plugins import get_site_plugins from ert.run_models import StatusEvents -from ert.run_models.everest_run_model import EverestExitCode, EverestRunModel +from ert.run_models.everest_run_model import EverestExitCode +from ert.run_models.model_factory import ( + EverestRunModel, + RunModelConfigs, + _instantiate_run_model, +) from everest.config import EverestConfig from everest.detached.everserver import ( ExperimentState, @@ -48,6 +51,9 @@ EverEndpoints, ) +if TYPE_CHECKING: + from ert.run_models.run_model import RunModel + router = APIRouter(prefix="/experiment_server", tags=["experiment_server"]) @@ -64,6 +70,9 @@ class ExperimentRunnerState: run_path: str | os.PathLike[str] | None = None storage_path: str | os.PathLike[str] | None = None start_time_unix: int | None = None + run_model: "RunModel | None" = dataclasses.field(default=None) + supports_rerunning_failed_realizations: bool = False + has_failed_realizations: bool = False _runs: dict[str, ExperimentRunnerState] = {} @@ -211,30 +220,80 @@ async def start_experiment( request: Request, background_tasks: BackgroundTasks, credentials: Annotated[HTTPBasicCredentials, Depends(security)], + rerun_from_run_id: str | None = None, ) -> JSONResponse: _log(request) _check_user(credentials) run_id = str(uuid.uuid4()) + if rerun_from_run_id is not None: + if rerun_from_run_id not in _runs: + raise HTTPException( + status_code=404, detail=f"Run '{rerun_from_run_id}' not found" + ) + source_run = _runs[rerun_from_run_id] + if source_run.run_model is None: + raise HTTPException( + status_code=400, + detail=f"Run '{rerun_from_run_id}' has no run model to rerun.", + ) + if not source_run.supports_rerunning_failed_realizations: + raise HTTPException( + status_code=400, + detail=f"Run '{rerun_from_run_id}' " + f"does not support rerunning failed realizations.", + ) + run_state = ExperimentRunnerState( + config_path=source_run.config_path, + run_path=source_run.run_path, + storage_path=source_run.storage_path, + run_model=source_run.run_model, + supports_rerunning_failed_realizations=source_run.supports_rerunning_failed_realizations, + ) + _runs[run_id] = run_state + runner = ExperimentRunner(None, run_id) + background_tasks.add_task(runner.run, rerun=True) + run_state.start_time_unix = int(time.time()) + return JSONResponse( + { + "run_id": run_id, + "supports_rerunning_failed_realizations": ( + run_state.supports_rerunning_failed_realizations + ), + } + ) run_state = ExperimentRunnerState() _runs[run_id] = run_state request_data = await request.json() - # The output of warnings is the task of the user interface, not - # of everserver. Therefore we suppress them here: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ConfigWarning) - config = EverestConfig.with_plugins(request_data) + adapter: TypeAdapter[RunModelConfigs] = TypeAdapter(RunModelConfigs) + config: RunModelConfigs | EverestConfig + if request_data.get("type") == "everest_config": + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ConfigWarning) + config = EverestConfig.with_plugins(request_data) + else: + config = adapter.validate_python(request_data) runner = ExperimentRunner(config, run_id) try: background_tasks.add_task(runner.run) - run_state.config_path = config.config_path - - run_state.run_path = config.simulation_dir - run_state.storage_path = config.output_dir - + if isinstance(config, EverestConfig): + run_state.config_path = config.config_file + run_state.run_path = config.output_dir + run_state.storage_path = str(config.storage_dir) + else: + run_state.config_path = config.user_config_file + run_state.run_path = config.runpath_config.runpath_format_string + run_state.storage_path = config.storage_path # Assume client and server is always in the same timezone # so disregard timestamps run_state.start_time_unix = int(time.time()) - return JSONResponse({"run_id": run_id}) + return JSONResponse( + { + "run_id": run_id, + "supports_rerunning_failed_realizations": ( + run_state.supports_rerunning_failed_realizations + ), + } + ) except Exception as e: run_state.status = ExperimentStatus( status=ExperimentState.failed, @@ -281,6 +340,71 @@ async def start_time( return Response(str(run.start_time_unix), status_code=200) +@router.post("/" + EverEndpoints.check_runpath) +async def check_runpath( + request: Request, + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> JSONResponse: + _log(request) + _check_user(credentials) + status_queue: SimpleQueue[StatusEvents] = SimpleQueue() + request_data = await request.json() + adapter: TypeAdapter[RunModelConfigs] = TypeAdapter(RunModelConfigs) + try: + config = adapter.validate_python(request_data) + run_model = _instantiate_run_model(config, status_queue) + try: + runpath_exists = run_model.check_if_runpath_exists() + return JSONResponse( + { + "runpath_exists": runpath_exists, + "num_existing": run_model.get_number_of_existing_runpaths(), + "num_active": run_model.get_number_of_active_realizations(), + } + ) + finally: + run_model._storage.close() + except Exception as e: + raise HTTPException( + status_code=422, detail=f"Could not check runpath: {e!s}" + ) from e + + +@router.post("/" + EverEndpoints.delete_runpaths) +async def delete_runpaths( + request: Request, + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> Response: + _log(request) + _check_user(credentials) + status_queue: SimpleQueue[StatusEvents] = SimpleQueue() + request_data = await request.json() + adapter: TypeAdapter[RunModelConfigs] = TypeAdapter(RunModelConfigs) + try: + config = adapter.validate_python(request_data) + run_model = _instantiate_run_model(config, status_queue) + try: + run_model.rm_run_path() + finally: + run_model._storage.close() + return Response("Runpaths deleted.", 200) + except Exception as e: + raise HTTPException( + status_code=422, detail=f"Could not delete runpaths: {e!s}" + ) from e + + +@router.get(f"/{EverEndpoints.has_failed_realizations}/{{run_id}}") +def has_failed_realizations_endpoint( + request: Request, + run: Annotated[ExperimentRunnerState, Depends(_get_run)], + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> JSONResponse: + _log(request) + _check_user(credentials) + return JSONResponse({"has_failed": run.has_failed_realizations}) + + @router.websocket(f"/{EverEndpoints.events}/{{run_id}}") async def websocket_endpoint(websocket: WebSocket, run_id: str) -> None: await websocket.accept() @@ -328,27 +452,30 @@ async def _get_event(subscriber_id: str, run_id: str) -> StatusEvents: class ExperimentRunner: def __init__( self, - everest_config: EverestConfig, + config: RunModelConfigs | None, run_id: str, ) -> None: super().__init__() - self._everest_config = everest_config + self._config = config self._run_id = run_id - async def run(self) -> None: + async def run(self, rerun: bool = False) -> None: run = _runs[self._run_id] status_queue: SimpleQueue[StatusEvents] = SimpleQueue() - run_model: EverestRunModel | None = None + run_model: RunModel | None = None try: - site_plugins = get_site_plugins() - with use_runtime_plugins(site_plugins): - run_model = EverestRunModel.create( - everest_config=self._everest_config, - experiment_name=f"EnOpt@{datetime.datetime.now().astimezone().isoformat(timespec='seconds')}", - target_ensemble="batch", - status_queue=status_queue, - runtime_plugins=site_plugins, + if rerun and run.run_model is not None: + run_model = run.run_model + run_model._status_queue = status_queue + else: + assert self._config is not None, ( + "ExperimentRunner.run() called without config for a fresh run" + ) + run_model = _instantiate_run_model(self._config, status_queue) + run.run_model = run_model + run.supports_rerunning_failed_realizations = ( + run_model.supports_rerunning_failed_realizations ) run.status = ExperimentStatus( message="Experiment started", status=ExperimentState.running @@ -362,10 +489,11 @@ async def run(self) -> None: else EvaluatorServerConfig(use_ipc_protocol=False) ), ) + cancelled = False while True: - if run.status.status == ExperimentState.stopped: + if run.status.status == ExperimentState.stopped and not cancelled: run_model.cancel() - raise UserCancelled("Optimization aborted") + cancelled = True try: item: StatusEvents = status_queue.get(block=False) except queue.Empty: @@ -381,16 +509,27 @@ async def run(self) -> None: for sub in list(run.subscribers.values()): await sub.is_done() break - await simulation_future - assert run_model.exit_code is not None - exp_status, msg = _get_optimization_status( - run_model.exit_code, - run.events, - ) - run.status = ExperimentStatus( - message=msg, - status=exp_status, - ) + try: + await simulation_future + except Exception: + if not cancelled: + raise + if not cancelled: + if isinstance(run_model, EverestRunModel): + assert run_model.exit_code is not None + exp_state, msg = _get_optimization_status( + run_model.exit_code, run.events + ) + run_status = ExperimentStatus( + message=msg, + status=exp_state, + ) + else: + run_status = ExperimentStatus( + message="Experiment completed.", + status=ExperimentState.completed, + ) + run.status = run_status except UserCancelled as e: logging.getLogger(EXPERIMENT_SERVER).info(f"User cancelled: {e}") except Exception as e: @@ -400,8 +539,13 @@ async def run(self) -> None: status=ExperimentState.failed, ) finally: - if run_model and run_model._experiment: - run_model._experiment.status = run.status + if run_model is not None: + run.has_failed_realizations = run_model.has_failed_realizations() + if ( + isinstance(run_model, EverestRunModel) + and run_model._experiment is not None + ): + run_model._experiment.status = run.status logging.getLogger(EXPERIMENT_SERVER).info( f"ExperimentRunner done. Items left in queue: {status_queue.qsize()}" diff --git a/src/ert/gui/experiments/experiment_client.py b/src/ert/gui/experiments/experiment_client.py index e389066a42a..e995b7411fd 100644 --- a/src/ert/gui/experiments/experiment_client.py +++ b/src/ert/gui/experiments/experiment_client.py @@ -7,7 +7,7 @@ import traceback from base64 import b64encode from http import HTTPStatus -from pathlib import Path +from typing import Any import requests from pydantic import ValidationError @@ -27,12 +27,12 @@ class ExperimentClient: def __init__( self, - run_id: str, url: str, cert_file: str, username: str, password: str, ssl_context: ssl.SSLContext, + run_id: str | None = None, ) -> None: self._run_id = run_id self._url = url @@ -60,6 +60,15 @@ def _http_post(self, endpoint: str) -> requests.Response: proxies={"http": None, "https": None}, # type: ignore ) + def _http_post_json(self, endpoint: str, body: dict[str, Any]) -> requests.Response: + return requests.post( + f"{self._url}/{endpoint}", + json=body, + verify=self._cert, + auth=(self._username, self._password), + proxies={"http": None, "https": None}, # type: ignore + ) + @property def config(self) -> dict[str, str]: return self._http_get(f"{EverEndpoints.config_path}/{self._run_id}").json() @@ -68,15 +77,53 @@ def config(self) -> dict[str, str]: def credentials(self) -> str: return b64encode(f"{self._username}:{self._password}".encode()).decode() + def check_runpath(self, config_json: dict[str, Any]) -> dict[str, Any]: + response = self._http_post_json(EverEndpoints.check_runpath, config_json) + response.raise_for_status() + return response.json() + + def delete_runpaths(self, config_json: dict[str, Any]) -> None: + response = self._http_post_json(EverEndpoints.delete_runpaths, config_json) + response.raise_for_status() + + def start_experiment(self, config_json: dict[str, Any]) -> tuple[str, bool]: + response = self._http_post_json(EverEndpoints.start_experiment, config_json) + response.raise_for_status() + data = response.json() + self._run_id = data["run_id"] + return self._run_id, data.get("supports_rerunning_failed_realizations", False) + + def rerun_failed(self) -> tuple[str, bool]: + assert self._run_id is not None, "No active run to rerun" + response = self._http_post( + f"{EverEndpoints.start_experiment}?rerun_from_run_id={self._run_id}" + ) + response.raise_for_status() + data = response.json() + self._run_id = data["run_id"] + return self._run_id, data.get("supports_rerunning_failed_realizations", False) + + def has_failed_realizations(self) -> bool: + assert self._run_id is not None, "No active run" + response = self._http_get( + f"{EverEndpoints.has_failed_realizations}/{self._run_id}" + ) + response.raise_for_status() + return bool(response.json().get("has_failed", False)) + def setup_event_queue_from_ws_endpoint( self, + event_queue: queue.SimpleQueue[StatusEvents] | None = None, refresh_interval: float = 0.01, open_timeout: float = 30, websocket_recv_timeout: float = 1.0, ) -> tuple[queue.SimpleQueue[StatusEvents], ErtThread]: - event_queue: queue.SimpleQueue[StatusEvents] = queue.SimpleQueue() + if event_queue is None: + event_queue = queue.SimpleQueue() + out_queue = event_queue def passthrough_ws_events() -> None: + assert self._run_id is not None, "No active run to stream events for" try: with connect( self._url.replace("https://", "wss://") @@ -93,7 +140,7 @@ def passthrough_ws_events() -> None: if message: try: event = status_event_from_json(message) - event_queue.put(event) + out_queue.put(event) except ValidationError as e: logger.error( "Error when processing event %s", exc_info=e @@ -106,14 +153,18 @@ def passthrough_ws_events() -> None: logger.debug(traceback.format_exc()) monitor_thread = ErtThread( - name="everest_gui_event_monitor", + name="ert_gui_event_monitor", target=passthrough_ws_events, daemon=True, ) - return event_queue, monitor_thread + return out_queue, monitor_thread - def create_run_model_api(self) -> RunModelAPI: + def create_run_model_api( + self, + experiment_name: str, + supports_rerunning_failed_realizations: bool, + ) -> RunModelAPI: def start_fn( evaluator_server_config: EvaluatorServerConfig, rerun_failed_realizations: bool = False, @@ -121,11 +172,11 @@ def start_fn( pass return RunModelAPI( - experiment_name=Path(self.config["config_path"]).name, - supports_rerunning_failed_realizations=False, + experiment_name=experiment_name, + supports_rerunning_failed_realizations=supports_rerunning_failed_realizations, start_simulations_thread=start_fn, cancel=self.stop, - has_failed_realizations=lambda: False, + has_failed_realizations=self.has_failed_realizations, ) def stop(self) -> None: diff --git a/src/ert/gui/experiments/experiment_panel.py b/src/ert/gui/experiments/experiment_panel.py index 4683ba200a2..1e68e68c57e 100644 --- a/src/ert/gui/experiments/experiment_panel.py +++ b/src/ert/gui/experiments/experiment_panel.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json +import ssl from collections import OrderedDict from pathlib import Path from queue import SimpleQueue @@ -11,7 +13,6 @@ from PyQt6.QtWidgets import ( QApplication, QCheckBox, - QDialog, QFrame, QHBoxLayout, QMessageBox, @@ -22,27 +23,25 @@ QWidget, ) -from _ert.threading import ErtThread -from ert.config import QueueSystem -from ert.ensemble_evaluator import EvaluatorServerConfig from ert.gui.detect_mode import is_dark_mode from ert.gui.ertnotifier import ErtNotifier from ert.gui.find_ert_info import find_ert_info from ert.gui.icon_utils import load_icon from ert.gui.summarypanel import SummaryPanel -from ert.run_models import RunModel, StatusEvents, create_model +from ert.run_models import RunModel, RunModelAPI, StatusEvents +from ert.run_models.model_factory import build_run_model_config from .combobox_with_description import QComboBoxWithDescription from .ensemble_experiment_panel import EnsembleExperimentPanel from .ensemble_information_filter_panel import EnsembleInformationFilterPanel from .ensemble_smoother_panel import EnsembleSmootherPanel from .evaluate_ensemble_panel import EvaluateEnsemblePanel +from .experiment_client import ExperimentClient from .experiment_config_panel import ExperimentConfigPanel from .manual_update_panel import ManualUpdatePanel from .multiple_data_assimilation_panel import MultipleDataAssimilationPanel from .run_dialog import RunDialog from .single_test_run_panel import SingleTestRunPanel -from .view.runpath_progress_widget import RunpathProgressWidget if TYPE_CHECKING: from ert.config import ErtConfig @@ -58,20 +57,6 @@ def create_md_table(kv: dict[str, str], output: str) -> str: return output -def get_simulation_thread( - model: Any, rerun_failed_realizations: bool = False, use_ipc_protocol: bool = False -) -> ErtThread: - evaluator_server_config = EvaluatorServerConfig(use_ipc_protocol=use_ipc_protocol) - - def run() -> None: - model.api.start_simulations_thread( - evaluator_server_config=evaluator_server_config, - rerun_failed_realizations=rerun_failed_realizations, - ) - - return ErtThread(name="ert_gui_simulation_thread", target=run, daemon=True) - - class ExperimentPanel(QWidget): experiment_type_changed = Signal(ExperimentConfigPanel) experiment_started = Signal(RunDialog) @@ -296,35 +281,59 @@ def get_experiment_arguments(self) -> Any: simulation_widget = self._experiment_widgets[self.get_current_experiment_type()] return simulation_widget.get_experiment_arguments() + def _create_experiment_client(self) -> ExperimentClient: + conn_info_path = Path(self.config.ens_path) / "storage_server.json" + conn_info: dict[str, Any] = json.loads( + conn_info_path.read_text(encoding="utf-8") + ) + url = conn_info["urls"][0] + cert_file = conn_info["cert"] + ssl_context = ssl.create_default_context() + ssl_context.load_verify_locations(cafile=cert_file) + return ExperimentClient( + url=f"{url}/experiment_server", + cert_file=cert_file, + username="__token__", + password=conn_info["authtoken"], + ssl_context=ssl_context, + ) + def run_experiment(self) -> None: args = self.get_experiment_arguments() QApplication.setOverrideCursor(Qt.CursorShape.WaitCursor) - event_queue: SimpleQueue[StatusEvents] = SimpleQueue() try: - model = create_model( - self.config, - args, - event_queue, + run_model_config = build_run_model_config(self.config, args) + config_json = run_model_config.model_dump(mode="json") + client = self._create_experiment_client() + except Exception as e: + QApplication.restoreOverrideCursor() + QMessageBox.warning( + self, + "ERROR: Failed to prepare experiment", + str(e), + QMessageBox.StandardButton.Ok, ) + return + + QApplication.restoreOverrideCursor() - except ValueError as e: + try: + runpath_check = client.check_runpath(config_json) + except Exception as e: QMessageBox.warning( self, - "ERROR: Failed to create experiment", + "ERROR: run_path check failed", str(e), QMessageBox.StandardButton.Ok, ) return - self._model = model - - QApplication.restoreOverrideCursor() - if model.check_if_runpath_exists(): + if runpath_check.get("runpath_exists"): + num_existing = runpath_check.get("num_existing", 0) + num_active = runpath_check.get("num_active", 0) msg_box = QMessageBox(self) msg_box.setObjectName("RUN_PATH_WARNING_BOX") - msg_box.setIcon(QMessageBox.Icon.Warning) - msg_box.setText("Run experiments") msg_box.setInformativeText( "ERT is running in an existing runpath.\n\n" @@ -333,55 +342,28 @@ def run_experiment(self) -> None: "might be overwritten.\n" "- Previously generated files might " "be used if not configured correctly.\n" - f"- {model.get_number_of_existing_runpaths()} out " - f"of {model.get_number_of_active_realizations()} realizations " + f"- {num_existing} out of {num_active} realizations " "are running in existing runpaths.\n" "Are you sure you want to continue?" ) - delete_runpath_checkbox = QCheckBox() delete_runpath_checkbox.setText("Delete run_path") msg_box.setCheckBox(delete_runpath_checkbox) - msg_box.setStandardButtons( QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No ) msg_box.setDefaultButton(QMessageBox.StandardButton.No) - msg_box.setWindowModality(Qt.WindowModality.ApplicationModal) - msg_box_res = msg_box.exec() - if msg_box_res == QMessageBox.StandardButton.No: - self._model._storage.close() + if msg_box.exec() == QMessageBox.StandardButton.No: return if delete_runpath_checkbox.checkState() == Qt.CheckState.Checked: - progress_dialog = QDialog(self) - progress_dialog.setObjectName("RUN_PATH_PROGRESS_DIALOG") - progress_dialog.setWindowTitle("Deleting runpaths") - progress_dialog.setWindowModality(Qt.WindowModality.ApplicationModal) - progress_layout = QVBoxLayout(progress_dialog) - progress_layout.setContentsMargins(0, 0, 0, 0) - - progress_widget = RunpathProgressWidget( - progress_dialog, - initial_status_text="Deleting runpaths...", - completed_action="deleted", - ) - progress_layout.addWidget(progress_widget) - progress_dialog.resize(420, 120) - progress_dialog.show() - QApplication.processEvents() - + QApplication.setOverrideCursor(Qt.CursorShape.WaitCursor) try: - model.rm_run_path( - progress_tracker=progress_widget, - # Force UI update during long deletion process - progress_callback=QApplication.processEvents, - ) + client.delete_runpaths(config_json) except OSError as e: - progress_dialog.close() - progress_dialog.deleteLater() + QApplication.restoreOverrideCursor() msg_box = QMessageBox(self) msg_box.setObjectName("RUN_PATH_ERROR_BOX") msg_box.setIcon(QMessageBox.Icon.Warning) @@ -394,20 +376,50 @@ def run_experiment(self) -> None: ) msg_box.setDefaultButton(QMessageBox.StandardButton.No) msg_box.setWindowModality(Qt.WindowModality.ApplicationModal) - msg_box_res = msg_box.exec() - if msg_box_res == QMessageBox.StandardButton.No: + if msg_box.exec() == QMessageBox.StandardButton.No: return else: - progress_dialog.close() - progress_dialog.deleteLater() + QApplication.restoreOverrideCursor() self.configuration_summary.log_summary( - args.mode, model.get_number_of_active_realizations() + args.mode, runpath_check.get("num_active", 0) + ) + + try: + _run_id, supports_rerunning = client.start_experiment(config_json) + except Exception as e: + QMessageBox.warning( + self, + "ERROR: Failed to start experiment", + str(e), + QMessageBox.StandardButton.Ok, + ) + return + + event_queue: SimpleQueue[StatusEvents] = SimpleQueue() + _, monitor_thread = client.setup_event_queue_from_ws_endpoint( + event_queue=event_queue + ) + + experiment_name = self.get_current_experiment_type().display_name() + + def start_fn( + evaluator_server_config: Any = None, + rerun_failed_realizations: bool = False, + ) -> None: + pass + + run_model_api = RunModelAPI( + experiment_name=experiment_name, + supports_rerunning_failed_realizations=supports_rerunning, + start_simulations_thread=start_fn, + cancel=client.stop, + has_failed_realizations=client.has_failed_realizations, ) self._dialog = RunDialog( f"Experiment - {self._config_file} {find_ert_info()}", - model.api, + run_model_api, event_queue, self._notifier, self.parent(), # type: ignore @@ -416,30 +428,35 @@ def run_experiment(self) -> None: storage_path=self._notifier.storage.path, ) self._dialog.queue_system.setText( - f"Queue system:\n{model.queue_config.queue_system.formatted_name}" + f"Queue system:\n{self.config.queue_config.queue_system.formatted_name}" ) self.experiment_started.emit(self._dialog) self._experiment_done = False self.run_button.setEnabled(self._experiment_done) - def start_simulation_thread(rerun_failed_realizations: bool = False) -> None: - simulation_thread = get_simulation_thread( - self._model, - rerun_failed_realizations, - use_ipc_protocol=self.config.queue_config.queue_system - == QueueSystem.LOCAL, + def do_rerun_failed() -> None: + try: + _new_run_id, _new_supports_rerunning = client.rerun_failed() + except Exception as e: + QMessageBox.warning( + self, + "ERROR: Failed to rerun", + str(e), + QMessageBox.StandardButton.Ok, + ) + return + _, rerun_thread = client.setup_event_queue_from_ws_endpoint( + event_queue=event_queue ) - self._dialog.setup_event_monitoring(rerun_failed_realizations) - simulation_thread.start() + self._dialog.setup_event_monitoring(rerun_failed_realizations=True) + rerun_thread.start() self._notifier.set_is_experiment_running(True) - def rerun_failed_realizations() -> None: - start_simulation_thread(rerun_failed_realizations=True) + self._dialog.rerun_failed_realizations_experiment.connect(do_rerun_failed) - self._dialog.rerun_failed_realizations_experiment.connect( - rerun_failed_realizations - ) - start_simulation_thread(rerun_failed_realizations=False) + self._dialog.setup_event_monitoring(rerun_failed_realizations=False) + monitor_thread.start() + self._notifier.set_is_experiment_running(True) def simulation_done_handler() -> None: self._experiment_done = True diff --git a/src/ert/logging/storage_log.conf b/src/ert/logging/storage_log.conf index b329bb53acf..d7f1fd74380 100644 --- a/src/ert/logging/storage_log.conf +++ b/src/ert/logging/storage_log.conf @@ -40,6 +40,8 @@ loggers: propagate: False ert.shared.status: level: INFO + ert.scheduler: + level: ERROR res: level: INFO res.config: diff --git a/src/ert/run_models/ensemble_experiment.py b/src/ert/run_models/ensemble_experiment.py index 6a4754973e0..945579c1e8a 100644 --- a/src/ert/run_models/ensemble_experiment.py +++ b/src/ert/run_models/ensemble_experiment.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import ClassVar +from typing import ClassVar, Literal from uuid import UUID from pydantic import PrivateAttr @@ -23,6 +23,7 @@ class EnsembleExperimentConfig(InitialEnsembleRunModelConfig): + type: Literal["ensemble_experiment"] = "ensemble_experiment" target_ensemble: str supports_rerunning_failed_realizations: ClassVar[bool] = True diff --git a/src/ert/run_models/ensemble_information_filter.py b/src/ert/run_models/ensemble_information_filter.py index 8303e8e80ad..2ffb454456d 100644 --- a/src/ert/run_models/ensemble_information_filter.py +++ b/src/ert/run_models/ensemble_information_filter.py @@ -2,6 +2,7 @@ import functools import logging +from typing import Literal from ert.analysis import enif_update from ert.run_models.ensemble_smoother import EnsembleSmoother @@ -16,10 +17,11 @@ class EnsembleInformationFilterConfig( InitialEnsembleRunModelConfig, UpdateRunModelConfig -): ... +): + type: Literal["ensemble_information_filter"] = "ensemble_information_filter" -class EnsembleInformationFilter(EnsembleSmoother, EnsembleInformationFilterConfig): +class EnsembleInformationFilter(EnsembleInformationFilterConfig, EnsembleSmoother): def update_ensemble_parameters( self, prior: Ensemble, posterior: Ensemble, weight: float ) -> None: diff --git a/src/ert/run_models/ensemble_smoother.py b/src/ert/run_models/ensemble_smoother.py index 8a34fc40430..659f17431c6 100644 --- a/src/ert/run_models/ensemble_smoother.py +++ b/src/ert/run_models/ensemble_smoother.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from typing import Literal import numpy as np from pydantic import PrivateAttr @@ -23,7 +24,8 @@ logger = logging.getLogger(__name__) -class EnsembleSmootherConfig(InitialEnsembleRunModelConfig, UpdateRunModelConfig): ... +class EnsembleSmootherConfig(InitialEnsembleRunModelConfig, UpdateRunModelConfig): + type: Literal["ensemble_smoother"] = "ensemble_smoother" class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmootherConfig): @@ -37,7 +39,7 @@ def run_experiment( ) -> None: self.log_at_startup() if rerun_failed_realizations: - raise ErtRunError("Ensemble Information Filter does not support restart") + raise ErtRunError("Ensemble Smoother does not support restart") self.run_workflows(fixtures=PreExperimentFixtures(random_seed=self.random_seed)) diff --git a/src/ert/run_models/evaluate_ensemble.py b/src/ert/run_models/evaluate_ensemble.py index 6904c30fe5f..951d6546752 100644 --- a/src/ert/run_models/evaluate_ensemble.py +++ b/src/ert/run_models/evaluate_ensemble.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal from uuid import UUID import numpy as np @@ -17,6 +17,7 @@ class EvaluateEnsembleConfig(RunModelConfig): + type: Literal["evaluate_ensemble"] = "evaluate_ensemble" ensemble_id: str supports_rerunning_failed_realizations: ClassVar[bool] = True diff --git a/src/ert/run_models/manual_update.py b/src/ert/run_models/manual_update.py index 3e2efb749a6..73b38079f90 100644 --- a/src/ert/run_models/manual_update.py +++ b/src/ert/run_models/manual_update.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any +from typing import Any, Literal from uuid import UUID from pydantic import PrivateAttr @@ -17,6 +17,7 @@ class ManualUpdateConfig(UpdateRunModelConfig): + type: Literal["manual_update"] = "manual_update" ensemble_id: str ert_templates: list[tuple[str, str]] diff --git a/src/ert/run_models/manual_update_enif.py b/src/ert/run_models/manual_update_enif.py index f4ecdbadeb5..1ceb29253fa 100644 --- a/src/ert/run_models/manual_update_enif.py +++ b/src/ert/run_models/manual_update_enif.py @@ -2,16 +2,21 @@ import functools import logging +from typing import Literal from ert.analysis import enif_update from ert.storage import Ensemble -from .manual_update import ManualUpdate +from .manual_update import ManualUpdate, ManualUpdateConfig logger = logging.getLogger(__name__) -class ManualUpdateEnIF(ManualUpdate): +class ManualUpdateEnIFConfig(ManualUpdateConfig): + type: Literal["manual_update_enif"] = "manual_update_enif" # type: ignore + + +class ManualUpdateEnIF(ManualUpdateEnIFConfig, ManualUpdate): @classmethod def name(cls) -> str: return "Manual EnIF update (Experimental)" diff --git a/src/ert/run_models/model_factory.py b/src/ert/run_models/model_factory.py index 746891dc2ff..7e2b553bc99 100644 --- a/src/ert/run_models/model_factory.py +++ b/src/ert/run_models/model_factory.py @@ -1,12 +1,15 @@ from __future__ import annotations import logging +from datetime import datetime from pathlib import Path from queue import SimpleQueue -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated import numpy as np +from pydantic import Field +from ert.base_model_context import use_runtime_plugins from ert.config import ( ConfigValidationError, ConfigWarning, @@ -25,7 +28,9 @@ MANUAL_UPDATE_MODE, TEST_RUN_MODE, ) +from ert.plugins import get_site_plugins from ert.validation import ActiveRange +from everest.config import EverestConfig from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig from .ensemble_information_filter import ( @@ -34,9 +39,10 @@ ) from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig from .evaluate_ensemble import EvaluateEnsemble, EvaluateEnsembleConfig +from .everest_run_model import EverestRunModel from .initial_ensemble_run_model import DictEncodedDataFrame from .manual_update import ManualUpdate, ManualUpdateConfig -from .manual_update_enif import ManualUpdateEnIF +from .manual_update_enif import ManualUpdateEnIF, ManualUpdateEnIFConfig from .multiple_data_assimilation import ( MultipleDataAssimilation, MultipleDataAssimilationConfig, @@ -51,6 +57,18 @@ from ert.run_models.event import StatusEvents +RunModelConfigs = Annotated[ + MultipleDataAssimilationConfig + | EnsembleSmootherConfig + | EnsembleInformationFilterConfig + | SingleTestRunConfig + | EnsembleExperimentConfig + | ManualUpdateConfig + | EvaluateEnsembleConfig + | EverestConfig, + Field(discriminator="type"), +] + logger = logging.getLogger(__name__) @@ -66,31 +84,62 @@ def create_model( "ensemble_size": config.runpath_config.num_realizations, }, ) + runmodel_config = build_run_model_config(config, args) + return _instantiate_run_model(runmodel_config, status_queue) + + +def build_run_model_config(config: ErtConfig, args: Namespace) -> RunModelConfigs: update_settings = config.analysis_config.observation_settings if args.mode == TEST_RUN_MODE: - return _setup_single_test_run(config, args, status_queue) + return _setup_single_test_run(config, args) if args.mode == ENSEMBLE_EXPERIMENT_MODE: - return _setup_ensemble_experiment(config, args, status_queue) + return _setup_ensemble_experiment(config, args) if args.mode == EVALUATE_ENSEMBLE_MODE: - return _setup_evaluate_ensemble(config, args, status_queue) + return _setup_evaluate_ensemble(config, args) if args.mode == ENSEMBLE_SMOOTHER_MODE: - return _setup_ensemble_smoother(config, args, update_settings, status_queue) + return _setup_ensemble_smoother(config, args, update_settings) if args.mode == ENIF_MODE: - return _setup_ensemble_information_filter( - config, args, update_settings, status_queue - ) + return _setup_ensemble_information_filter(config, args, update_settings) if args.mode == ES_MDA_MODE: - return _setup_multiple_data_assimilation( - config, args, update_settings, status_queue - ) + return _setup_multiple_data_assimilation(config, args, update_settings) if args.mode == MANUAL_UPDATE_MODE: - return _setup_manual_update(config, args, update_settings, status_queue) + return _setup_manual_update(config, args, update_settings) if args.mode == MANUAL_ENIF_UPDATE_MODE: - return _setup_manual_update_enif(config, args, update_settings, status_queue) + return _setup_manual_update_enif(config, args, update_settings) raise NotImplementedError(f"Run type not supported {args.mode}") +def _instantiate_run_model( + runmodel_config: RunModelConfigs, + status_queue: SimpleQueue[StatusEvents], +) -> RunModel: + """Instantiate a RunModel from a config object.""" + if isinstance(runmodel_config, EverestConfig): + site_plugins = get_site_plugins() + with use_runtime_plugins(site_plugins): + return EverestRunModel.create( + everest_config=runmodel_config, + experiment_name=f"EnOpt@{datetime.now().astimezone().isoformat(timespec='seconds')}", + target_ensemble="batch", + status_queue=status_queue, + runtime_plugins=site_plugins, + ) + + model_map: dict[str, type[RunModel]] = { + "single_test_run": SingleTestRun, + "ensemble_experiment": EnsembleExperiment, + "evaluate_ensemble": EvaluateEnsemble, + "ensemble_smoother": EnsembleSmoother, + "ensemble_information_filter": EnsembleInformationFilter, + "multiple_data_assimilation": MultipleDataAssimilation, + "manual_update": ManualUpdate, + "manual_update_enif": ManualUpdateEnIF, + } + model_cls = model_map[runmodel_config.type] + return model_cls(**runmodel_config.model_dump(), status_queue=status_queue) + + def _merge_parameters( design_matrix: DesignMatrix | None, parameter_configs: list[ParameterConfig], @@ -116,8 +165,7 @@ def _merge_parameters( def _setup_single_test_run( config: ErtConfig, args: Namespace, - status_queue: SimpleQueue[StatusEvents], -) -> SingleTestRun: +) -> SingleTestRunConfig: experiment_name = ( "single-test-run" if args.experiment_name is None else args.experiment_name ) @@ -132,7 +180,7 @@ def _setup_single_test_run( parameter_configs=config.ensemble_config.parameter_configuration, ) - runmodel_config = SingleTestRunConfig( + return SingleTestRunConfig( random_seed=config.random_seed, runpath_file=config.runpath_file, active_realizations=[True], @@ -158,11 +206,6 @@ def _setup_single_test_run( shape_registry=config.shape_registry, ) - return SingleTestRun( - **runmodel_config.model_dump(), - status_queue=status_queue, - ) - def validate_minimum_realizations( config: ErtConfig, active_realizations: list[bool] @@ -181,8 +224,7 @@ def validate_minimum_realizations( def _setup_ensemble_experiment( config: ErtConfig, args: Namespace, - status_queue: SimpleQueue[StatusEvents], -) -> EnsembleExperiment: +) -> EnsembleExperimentConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) experiment_name = args.experiment_name @@ -193,7 +235,7 @@ def _setup_ensemble_experiment( parameter_configs=config.ensemble_config.parameter_configuration, ) - runmodel_config = EnsembleExperimentConfig( + return EnsembleExperimentConfig( random_seed=config.random_seed, runpath_file=config.runpath_file, active_realizations=active_realizations, @@ -219,20 +261,14 @@ def _setup_ensemble_experiment( shape_registry=config.shape_registry, ) - return EnsembleExperiment( - **runmodel_config.model_dump(), - status_queue=status_queue, - ) - def _setup_evaluate_ensemble( config: ErtConfig, args: Namespace, - status_queue: SimpleQueue[StatusEvents], -) -> EvaluateEnsemble: +) -> EvaluateEnsembleConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) - runmodel_config = EvaluateEnsembleConfig( + return EvaluateEnsembleConfig( random_seed=config.random_seed, active_realizations=active_realizations, ensemble_id=args.ensemble_id, @@ -250,7 +286,6 @@ def _setup_evaluate_ensemble( log_path=config.analysis_config.log_path, shape_registry=config.shape_registry, ) - return EvaluateEnsemble(**runmodel_config.model_dump(), status_queue=status_queue) def _get_and_validate_active_realizations_list( @@ -285,12 +320,11 @@ def _setup_manual_update( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> ManualUpdate: +) -> ManualUpdateConfig: active_realizations = _realizations(args, config.runpath_config.num_realizations) validate_minimum_realizations(config, active_realizations.tolist()) - runmodel_config = ManualUpdateConfig( + return ManualUpdateConfig( random_seed=config.random_seed, active_realizations=active_realizations.tolist(), ensemble_id=args.ensemble_id, @@ -310,37 +344,28 @@ def _setup_manual_update( hooked_workflows=config.hooked_workflows, log_path=config.analysis_config.log_path, ert_templates=config.ert_templates, - observations=config.observation_declarations, shape_registry=config.shape_registry, ) - return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue) def _setup_manual_update_enif( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> ManualUpdate: +) -> ManualUpdateEnIFConfig: active_realizations = _realizations(args, config.runpath_config.num_realizations) - return ManualUpdateEnIF( + return ManualUpdateEnIFConfig( random_seed=config.random_seed, active_realizations=active_realizations.tolist(), ensemble_id=args.ensemble_id, minimum_required_realizations=config.analysis_config.minimum_required_realizations, target_ensemble=args.target_ensemble, - config=config, storage_path=config.ens_path, queue_config=config.queue_config, analysis_settings=config.analysis_config.es_settings, update_settings=update_settings, - status_queue=status_queue, runpath_file=config.runpath_file, - design_matrix=config.analysis_config.design_matrix, - parameter_configuration=config.ensemble_config.parameter_configuration, - response_configuration=config.ensemble_config.response_configuration, - derived_response_configuration=config.ensemble_config.derived_response_configuration, ert_templates=config.ert_templates, user_config_file=Path(config.user_config_file), env_vars=config.env_vars, @@ -350,7 +375,6 @@ def _setup_manual_update_enif( substitutions=config.substitutions, hooked_workflows=config.hooked_workflows, log_path=config.analysis_config.log_path, - observations=config.observation_declarations, shape_registry=config.shape_registry, ) @@ -359,8 +383,7 @@ def _setup_ensemble_smoother( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> EnsembleSmoother: +) -> EnsembleSmootherConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) if sum(active_realizations) < 2: @@ -374,7 +397,7 @@ def _setup_ensemble_smoother( require_updateable_param=True, ) - runmodel_config = EnsembleSmootherConfig( + return EnsembleSmootherConfig( target_ensemble=args.target_ensemble, experiment_name=getattr(args, "experiment_name", ""), active_realizations=active_realizations, @@ -401,15 +424,13 @@ def _setup_ensemble_smoother( observations=config.observation_declarations, shape_registry=config.shape_registry, ) - return EnsembleSmoother(**runmodel_config.model_dump(), status_queue=status_queue) def _setup_ensemble_information_filter( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> EnsembleInformationFilter: +) -> EnsembleInformationFilterConfig: active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) if sum(active_realizations) < 2: @@ -422,7 +443,7 @@ def _setup_ensemble_information_filter( parameter_configs=config.ensemble_config.parameter_configuration, ) - runmodel_config = EnsembleInformationFilterConfig( + return EnsembleInformationFilterConfig( target_ensemble=args.target_ensemble, experiment_name=getattr(args, "experiment_name", ""), active_realizations=active_realizations, @@ -449,9 +470,6 @@ def _setup_ensemble_information_filter( observations=config.observation_declarations, shape_registry=config.shape_registry, ) - return EnsembleInformationFilter( - **runmodel_config.model_dump(), status_queue=status_queue - ) def _determine_restart_info(args: Namespace) -> tuple[bool, str | None]: @@ -477,8 +495,7 @@ def _setup_multiple_data_assimilation( config: ErtConfig, args: Namespace, update_settings: ObservationSettings, - status_queue: SimpleQueue[StatusEvents], -) -> MultipleDataAssimilation: +) -> MultipleDataAssimilationConfig: restart_run, prior_ensemble = _determine_restart_info(args) active_realizations = _get_and_validate_active_realizations_list(args, config) validate_minimum_realizations(config, active_realizations) @@ -493,7 +510,7 @@ def _setup_multiple_data_assimilation( require_updateable_param=True, ) - runmodel_config = MultipleDataAssimilationConfig( + return MultipleDataAssimilationConfig( random_seed=config.random_seed, active_realizations=active_realizations, target_ensemble=_iterative_ensemble_format(args), @@ -523,9 +540,6 @@ def _setup_multiple_data_assimilation( observations=config.observation_declarations, shape_registry=config.shape_registry, ) - return MultipleDataAssimilation( - **runmodel_config.model_dump(), status_queue=status_queue - ) def _realizations( diff --git a/src/ert/run_models/multiple_data_assimilation.py b/src/ert/run_models/multiple_data_assimilation.py index 736aa4b3108..6cb70554eb7 100644 --- a/src/ert/run_models/multiple_data_assimilation.py +++ b/src/ert/run_models/multiple_data_assimilation.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal from uuid import UUID from pydantic import PrivateAttr @@ -31,6 +31,7 @@ class MultipleDataAssimilationConfig( InitialEnsembleRunModelConfig, UpdateRunModelConfig ): + type: Literal["multiple_data_assimilation"] = "multiple_data_assimilation" default_weights: ClassVar[str] = "4, 2, 1" restart_run: bool prior_ensemble_id: str | None diff --git a/src/ert/run_models/single_test_run.py b/src/ert/run_models/single_test_run.py index 9b14a67a174..2e880c517e0 100644 --- a/src/ert/run_models/single_test_run.py +++ b/src/ert/run_models/single_test_run.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Literal + from pydantic import Field from ert.run_models import EnsembleExperiment @@ -10,11 +12,12 @@ class SingleTestRunConfig(EnsembleExperimentConfig): + type: Literal["single_test_run"] = "single_test_run" # type: ignore active_realizations: list[bool] = Field(default_factory=lambda: [True]) minimum_required_realizations: int = 1 -class SingleTestRun(EnsembleExperiment, SingleTestRunConfig): +class SingleTestRun(SingleTestRunConfig, EnsembleExperiment): """ Single test is equivalent to EnsembleExperiment, in that it samples the prior and evaluates it.
There are two key differences:
diff --git a/src/everest/config/everest_config.py b/src/everest/config/everest_config.py index bb876cded74..113d0d797c5 100644 --- a/src/everest/config/everest_config.py +++ b/src/everest/config/everest_config.py @@ -10,6 +10,7 @@ from typing import ( Annotated, Any, + Literal, Optional, Self, TextIO, @@ -179,6 +180,7 @@ def _find_loc(loc: tuple[int | str, ...] | None, file_content: list[str]) -> int class EverestConfig(BaseModelWithContextSupport): + type: Literal["everest_config"] = "everest_config" controls: Annotated[list[ControlConfig], AfterValidator(unique_items)] = Field( description=dedent( """ @@ -1032,7 +1034,7 @@ def result_names(self) -> list[str]: def to_dict(self) -> dict[str, Any]: the_dict = self.model_dump(exclude_none=True, exclude_unset=True) - + the_dict["type"] = self.type if "config_path" in the_dict: the_dict["config_path"] = str(the_dict["config_path"]) diff --git a/src/everest/gui/main_window.py b/src/everest/gui/main_window.py index a407ccc5afe..33a81c7c3bb 100644 --- a/src/everest/gui/main_window.py +++ b/src/everest/gui/main_window.py @@ -70,7 +70,10 @@ def run(self) -> None: title = Path(config["config_path"]).name self.setWindowTitle(f"EVEREST - {title}") - run_model_api = client.create_run_model_api() + run_model_api = client.create_run_model_api( + experiment_name=title, + supports_rerunning_failed_realizations=False, + ) event_queue, event_monitor_thread = client.setup_event_queue_from_ws_endpoint( refresh_interval=0.02, open_timeout=40, websocket_recv_timeout=1.0 ) diff --git a/src/everest/strings.py b/src/everest/strings.py index 5cacb641696..4d81add53d1 100644 --- a/src/everest/strings.py +++ b/src/everest/strings.py @@ -34,3 +34,6 @@ class EverEndpoints(StrEnum): runs = "runs" status = "status" events = "events" + check_runpath = "check_runpath" + delete_runpaths = "delete_runpaths" + has_failed_realizations = "has_failed_realizations" diff --git a/tests/conftest.py b/tests/conftest.py index 8f619fab16f..f477b3b2e96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,39 @@ import os +import shutil +from pathlib import Path from unittest.mock import patch import pytest +from ert.config import ErtConfig from ert.plugins import ErtRuntimePlugins +def source_dir() -> Path: + src = Path("@CMAKE_CURRENT_SOURCE_DIR@/../..") + if src.is_dir(): + return src.relative_to(Path.cwd()) + + # If the file was not correctly configured by cmake, look for the source + # folder, assuming the build folder is inside the source folder. + current_path = Path(__file__) + while current_path != Path("/"): + if (current_path / ".git").is_dir(): + return current_path + # This is to find root dir for git worktrees + elif (current_path / ".git").is_file(): + with (current_path / ".git").open(encoding="utf-8") as f: + for line in f: + if "gitdir:" in line: + return current_path + + current_path = current_path.parent + raise RuntimeError("Cannot find the source folder") + + +SOURCE_DIR: Path = source_dir() + + def pytest_addoption(parser): parser.addoption( "--eclipse-simulator", @@ -138,3 +166,26 @@ def ErtRuntimePluginsWithNoQueueOptions(**kwargs): ErtRuntimePluginsWithNoQueueOptions, ): yield + + +@pytest.fixture(scope="session", name="source_root") +def fixture_source_root(): + return SOURCE_DIR + + +@pytest.fixture(name="setup_case") +def fixture_setup_case(tmp_path_factory, source_root, monkeypatch): + def copy_case(path, config_file): + tmp_path = tmp_path_factory.mktemp(path.replace("/", "-")) + shutil.copytree( + os.path.join(source_root, "test-data/ert", path), tmp_path / "test_data" + ) + monkeypatch.chdir(tmp_path / "test_data") + return ErtConfig.from_file(config_file) + + return copy_case + + +@pytest.fixture +def poly_case(setup_case): + return setup_case("poly_example", "poly.ert") diff --git a/tests/ert/conftest.py b/tests/ert/conftest.py index 00ffd24876e..50d38debae9 100644 --- a/tests/ert/conftest.py +++ b/tests/ert/conftest.py @@ -36,8 +36,6 @@ ) from ert.storage import open_storage -from .utils import SOURCE_DIR - st.register_type_strategy(Path, st.builds(Path, st.text().map(lambda x: "/tmp/" + x))) @@ -97,11 +95,6 @@ def _qt_add_search_paths(qapp): ) -@pytest.fixture(scope="session", name="source_root") -def fixture_source_root(): - return SOURCE_DIR - - @pytest.fixture(scope="class") def class_source_root(request, source_root): request.cls.SOURCE_ROOT = source_root @@ -121,24 +114,6 @@ def maximize_ulimits(): resource.setrlimit(resource.RLIMIT_NOFILE, limits) -@pytest.fixture(name="setup_case") -def fixture_setup_case(tmp_path_factory, source_root, monkeypatch): - def copy_case(path, config_file): - tmp_path = tmp_path_factory.mktemp(path.replace("/", "-")) - shutil.copytree( - os.path.join(source_root, "test-data/ert", path), tmp_path / "test_data" - ) - monkeypatch.chdir(tmp_path / "test_data") - return ErtConfig.from_file(config_file) - - return copy_case - - -@pytest.fixture -def poly_case(setup_case): - return setup_case("poly_example", "poly.ert") - - @pytest.fixture def snake_oil_case_storage(copy_snake_oil_case_storage): with warnings.catch_warnings(): diff --git a/tests/ert/ui_tests/gui/conftest.py b/tests/ert/ui_tests/gui/conftest.py index cc4990ec985..652b8586177 100644 --- a/tests/ert/ui_tests/gui/conftest.py +++ b/tests/ert/ui_tests/gui/conftest.py @@ -35,7 +35,7 @@ from ert.gui.tools.manage_experiments.storage_widget import AddWidget, StorageWidget from ert.plugins import get_site_plugins from ert.run_models import EnsembleExperiment, MultipleDataAssimilation -from ert.storage import Storage +from ert.services import ErtServerController from tests.ert.handle_run_path_dialog import handle_run_path_dialog DEFAULT_NUM_REALIZATIONS = 10 @@ -94,15 +94,19 @@ def _new_poly_example( @contextmanager -def _open_main_window(path) -> Iterator[tuple[ErtMainWindow, Storage, ErtConfig]]: +def _open_main_window(path) -> Iterator[tuple[ErtMainWindow, str, ErtConfig]]: args_mock = Mock() args_mock.config = str(path) site_plugins = get_site_plugins() with use_runtime_plugins(site_plugins): config = ErtConfig.with_plugins(site_plugins).from_file(path) + ens_path = Path(config.ens_path).absolute() + ens_path.mkdir(parents=True, exist_ok=True) with ( add_gui_log_handler() as log_handler, + ErtServerController.init_service(project=ens_path) as server, ): + server.wait_until_ready() gui = _setup_main_window(config, args_mock, log_handler, config.ens_path) yield gui, config.ens_path, config gui.close() diff --git a/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py b/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py new file mode 100644 index 00000000000..05ea8506623 --- /dev/null +++ b/tests/ert/unit_tests/dark_storage/test_experiment_server_new_endpoints.py @@ -0,0 +1,271 @@ +"""Unit tests for the new experiment_server endpoints: +- POST /check_runpath +- POST /delete_runpaths +- POST /start_experiment?rerun_from_run_id={run_id} +- GET /has_failed_realizations/{run_id} +""" + +from __future__ import annotations + +import shutil +from argparse import Namespace +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from ert.config import ErtConfig +from ert.dark_storage.app import app +from ert.dark_storage.endpoints.experiment_server import ( + ExperimentRunnerState, + _runs, +) +from ert.mode_definitions import ENSEMBLE_EXPERIMENT_MODE +from ert.run_models.model_factory import build_run_model_config +from everest.strings import EverEndpoints + +_TOKEN = "test-token" +_AUTH = ("__token__", _TOKEN) + +check_runpath_url = f"/experiment_server/{EverEndpoints.check_runpath}" +_DELETE_RUNPATHS_URL = f"/experiment_server/{EverEndpoints.delete_runpaths}" + + +@pytest.fixture +def experiment_server_client(monkeypatch): + """TestClient with ERT_STORAGE_TOKEN set.""" + original = dict(_runs) + monkeypatch.setenv("ERT_STORAGE_TOKEN", _TOKEN) + with TestClient(app) as client: + yield client + _runs.clear() + _runs.update(original) + + +@pytest.fixture +def poly_run_model_config(copy_poly_case): + """Returns a serialized EnsembleExperiment RunModelConfig for poly_example.""" + config = ErtConfig.from_file("poly.ert") + args = Namespace( + mode=ENSEMBLE_EXPERIMENT_MODE, + realizations="0,1", + experiment_name="preflight_test", + current_ensemble="default", + ) + return build_run_model_config(config, args).model_dump(mode="json") + + +@pytest.fixture +def copy_poly_case(tmp_path, source_root, monkeypatch): + poly_dir = tmp_path / "poly_example" + shutil.copytree( + source_root / "test-data" / "ert" / "poly_example", + poly_dir, + ignore=shutil.ignore_patterns("*ipynb", "poly_out", "storage", "logs"), + ) + monkeypatch.chdir(poly_dir) + return poly_dir + + +def test_that_check_runpath_returns_false_when_no_runpath_exists( + experiment_server_client, poly_run_model_config +): + response = experiment_server_client.post( + check_runpath_url, json=poly_run_model_config, auth=_AUTH + ) + assert response.status_code == 200 + data = response.json() + assert data["runpath_exists"] is False + assert data["num_existing"] == 0 + assert data["num_active"] == 2 + + +def test_that_check_runpath_returns_true_when_runpath_exists( + experiment_server_client, poly_run_model_config, tmp_path +): + # Create the runpath directories that the config expects + + runpath_format = poly_run_model_config["runpath_config"]["runpath_format_string"] + # Create realization 0, iter 0 directory + runpath_0 = ( + Path(runpath_format % (0, 0)) + if "%d" in runpath_format + else Path(runpath_format.replace("", "0").replace("", "0")) + ) + runpath_0.mkdir(parents=True, exist_ok=True) + + response = experiment_server_client.post( + check_runpath_url, json=poly_run_model_config, auth=_AUTH + ) + assert response.status_code == 200 + data = response.json() + assert data["runpath_exists"] is True + + +def test_that_check_runpath_requires_authentication( + experiment_server_client, poly_run_model_config +): + response = experiment_server_client.post( + check_runpath_url, json=poly_run_model_config + ) + assert response.status_code == 401 + + +def test_that_check_runpath_returns_422_for_invalid_config( + experiment_server_client, +): + response = experiment_server_client.post( + check_runpath_url, json={"type": "invalid_type"}, auth=_AUTH + ) + assert response.status_code == 422 + + +def test_that_delete_runpaths_removes_existing_run_directories( + experiment_server_client, poly_run_model_config, tmp_path +): + runpath_format = poly_run_model_config["runpath_config"]["runpath_format_string"] + # Substitute template placeholders — use ERT's realization/iter format + runpath_0 = Path( + runpath_format.replace("", "0").replace("", "0") + if "" in runpath_format + else runpath_format % (0, 0) + ) + runpath_0.mkdir(parents=True, exist_ok=True) + assert runpath_0.exists() + + response = experiment_server_client.post( + _DELETE_RUNPATHS_URL, json=poly_run_model_config, auth=_AUTH + ) + assert response.status_code == 200 + assert not runpath_0.exists() + + +def test_that_delete_runpaths_requires_authentication( + experiment_server_client, poly_run_model_config +): + response = experiment_server_client.post( + _DELETE_RUNPATHS_URL, json=poly_run_model_config + ) + assert response.status_code == 401 + + +def test_that_has_failed_realizations_returns_false_when_no_realizations_failed( + experiment_server_client, +): + run_id = "test-run-no-failed" + state = ExperimentRunnerState(has_failed_realizations=False) + _runs[run_id] = state + + response = experiment_server_client.get( + f"/experiment_server/{EverEndpoints.has_failed_realizations}/{run_id}", + auth=_AUTH, + ) + assert response.status_code == 200 + assert response.json() == {"has_failed": False} + + +def test_that_has_failed_realizations_returns_true_when_some_realizations_failed( + experiment_server_client, +): + run_id = "test-run-with-failed" + state = ExperimentRunnerState(has_failed_realizations=True) + _runs[run_id] = state + + response = experiment_server_client.get( + f"/experiment_server/{EverEndpoints.has_failed_realizations}/{run_id}", + auth=_AUTH, + ) + assert response.status_code == 200 + assert response.json() == {"has_failed": True} + + +def test_that_has_failed_realizations_returns_404_for_unknown_run_id( + experiment_server_client, +): + response = experiment_server_client.get( + f"/experiment_server/{EverEndpoints.has_failed_realizations}/nonexistent-id", + auth=_AUTH, + ) + assert response.status_code == 404 + + +def test_that_rerun_failed_creates_new_run_with_same_model( + experiment_server_client, +): + mock_run_model = MagicMock() + mock_run_model.supports_rerunning_failed_realizations = True + + run_id = "original-run-id" + state = ExperimentRunnerState( + run_model=mock_run_model, + supports_rerunning_failed_realizations=True, + config_path="/path/to/config.ert", + run_path="/path/to/runpath", + storage_path="/path/to/storage", + ) + _runs[run_id] = state + + with patch( + "ert.dark_storage.endpoints.experiment_server.ExperimentRunner.run" + ) as mock_run: + mock_run.return_value = None + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": run_id}, + auth=_AUTH, + ) + + assert response.status_code == 200 + data = response.json() + new_run_id = data["run_id"] + assert new_run_id != run_id + assert new_run_id in _runs + assert _runs[new_run_id].run_model is mock_run_model + assert data["supports_rerunning_failed_realizations"] is True + + +def test_that_rerun_failed_returns_400_when_run_model_is_missing( + experiment_server_client, +): + run_id = "run-without-model" + state = ExperimentRunnerState(run_model=None) + _runs[run_id] = state + + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": run_id}, + auth=_AUTH, + ) + assert response.status_code == 400 + + +def test_that_rerun_failed_returns_400_when_rerun_is_not_supported( + experiment_server_client, +): + mock_run_model = MagicMock() + + run_id = "run-no-rerun-support" + state = ExperimentRunnerState( + run_model=mock_run_model, + supports_rerunning_failed_realizations=False, + ) + _runs[run_id] = state + + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": run_id}, + auth=_AUTH, + ) + assert response.status_code == 400 + + +def test_that_rerun_failed_returns_404_for_unknown_run_id( + experiment_server_client, +): + response = experiment_server_client.post( + f"/experiment_server/{EverEndpoints.start_experiment}", + params={"rerun_from_run_id": "nonexistent-id"}, + auth=_AUTH, + ) + assert response.status_code == 404 diff --git a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py index a827de38894..c74e8a143a9 100644 --- a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py +++ b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py @@ -1,6 +1,7 @@ import tempfile from pathlib import Path from queue import SimpleQueue +from threading import Thread from unittest.mock import MagicMock, Mock, patch import pandas as pd @@ -53,25 +54,12 @@ StartingTotalRunPathCreationEvent, ) from ert.scheduler.job import Job +from ert.services import ErtServerController from tests.ert import SnapshotBuilder from tests.ert.handle_run_path_dialog import handle_run_path_dialog from tests.ert.ui_tests.gui.conftest import wait_for_child -@pytest.fixture -def event_queue(events): - async def _add_event(self, *_): - for event in events: - self.send_event(event) - return [0] - - with patch( - "ert.run_models.run_model.RunModel.run_ensemble_evaluator_async", - _add_event, - ): - yield - - @pytest.fixture def event_queue_large_snapshot(large_snapshot): events = [ @@ -125,7 +113,7 @@ def mock_set_env_key(): @pytest.fixture -def run_dialog(qtbot: QtBot, use_tmpdir, mock_set_env_key, monkeypatch): +def run_dialog(events, qtbot: QtBot, use_tmpdir, mock_set_env_key, monkeypatch): config_file = "minimal_config.ert" monkeypatch.setattr("ert.scheduler.Scheduler.BATCH_KILLING_INTERVAL", 0.01) monkeypatch.setattr( @@ -148,15 +136,44 @@ def run_dialog(qtbot: QtBot, use_tmpdir, mock_set_env_key, monkeypatch): simulation_settings._experiment_name_field.setText("new_experiment_name") run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") assert run_experiment - qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) - qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) + + def _make_mock_client(): + mock_client = MagicMock() + mock_client.check_runpath.return_value = { + "runpath_exists": False, + "num_existing": 0, + "num_active": 1, + } + mock_client.start_experiment.return_value = ("mock-run-id", False) + + def _setup_event_queue(event_queue=None, **_kwargs): + if event_queue is None: + event_queue = SimpleQueue() + + def _feed_events(): + for event in events: + event_queue.put(event) + + thread = Thread(target=_feed_events, daemon=True) + return event_queue, thread + + mock_client.setup_event_queue_from_ws_endpoint.side_effect = _setup_event_queue + return mock_client + + with patch.object( + ExperimentPanel, + "_create_experiment_client", + side_effect=_make_mock_client, + ): + qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) + qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) run_dialog = gui.findChild(RunDialog) assert run_dialog return run_dialog @pytest.mark.usefixtures("use_tmpdir") -@pytest.mark.timeout(10) +@pytest.mark.timeout(60) @pytest.mark.slow def test_that_terminating_experiment_shows_a_confirmation_dialog( qtbot: QtBot, monkeypatch @@ -192,48 +209,51 @@ def test_that_terminating_experiment_shows_a_confirmation_dialog( ) args_mock = Mock() args_mock.config = config_file - gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") - qtbot.addWidget(gui) - experiment_panel = gui.findChild(ExperimentPanel) - assert experiment_panel - simulation_mode_combo = experiment_panel.findChild(QComboBox) - assert simulation_mode_combo - simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) - simulation_settings = gui.findChild(EnsembleExperimentPanel) - simulation_settings._experiment_name_field.setText("new_experiment_name") - run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") - assert run_experiment - qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) - qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) - run_dialog = gui.findChild(RunDialog) - assert run_dialog - kill_button = run_dialog.kill_button - with qtbot.waitSignal(run_dialog.experiment_done, timeout=10000): - # Wait for ensemble to start evaluating before cancelling - _ = wait_for_child(run_dialog, qtbot, RealizationWidget) - - def handle_dialog(): - terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) - dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() - yes_button = next(b for b in dialog_buttons if "Yes" in b.text()) - qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) - - QTimer.singleShot(100, handle_dialog) - assert kill_button.isEnabled() + storage_path = Path("storage") + storage_path.mkdir() + with ErtServerController.init_service(project=storage_path.absolute()): + gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") + qtbot.addWidget(gui) + experiment_panel = gui.findChild(ExperimentPanel) + assert experiment_panel + simulation_mode_combo = experiment_panel.findChild(QComboBox) + assert simulation_mode_combo + simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) + simulation_settings = gui.findChild(EnsembleExperimentPanel) + simulation_settings._experiment_name_field.setText("new_experiment_name") + run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") + assert run_experiment + qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) + qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=30000) + run_dialog = gui.findChild(RunDialog) + assert run_dialog + kill_button = run_dialog.kill_button + with qtbot.waitSignal(run_dialog.experiment_done, timeout=30000): + # Wait for ensemble to start evaluating before cancelling + _ = wait_for_child(run_dialog, qtbot, RealizationWidget) + + def handle_dialog(): + terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) + dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() + yes_button = next(b for b in dialog_buttons if "Yes" in b.text()) + qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) + + QTimer.singleShot(100, handle_dialog) + assert kill_button.isEnabled() + assert kill_button.text() == "Terminate experiment" + qtbot.mouseClick(run_dialog.kill_button, Qt.MouseButton.LeftButton) + assert not kill_button.isEnabled() + assert kill_button.text() == "Terminating" + wait_for_child(run_dialog, qtbot, Suggestor) + assert run_dialog.fail_msg_box is not None assert kill_button.text() == "Terminate experiment" - qtbot.mouseClick(run_dialog.kill_button, Qt.MouseButton.LeftButton) assert not kill_button.isEnabled() - assert kill_button.text() == "Terminating" - wait_for_child(run_dialog, qtbot, Suggestor) - assert run_dialog.fail_msg_box is not None - assert kill_button.text() == "Terminate experiment" - assert not kill_button.isEnabled() - assert ( - "Experiment cancelled by user during evaluation" - in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") - .findChild(QLabel) - .text() - ) + assert ( + "Experiment cancelled by user during evaluation" + in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") + .findChild(QLabel) + .text() + ) @pytest.mark.slow @@ -440,7 +460,7 @@ def test_large_snapshot( ), ], ) -def test_run_dialog(events, event_queue, tab_widget_count, qtbot: QtBot, run_dialog): +def test_run_dialog(events, tab_widget_count, qtbot: QtBot, run_dialog): qtbot.waitUntil( lambda: run_dialog._tab_widget.count() == tab_widget_count, timeout=5000 ) @@ -514,7 +534,7 @@ def test_run_dialog(events, event_queue, tab_widget_count, qtbot: QtBot, run_dia ], ) def test_run_dialog_memory_usage_showing( - events, event_queue, tab_widget_count, qtbot: QtBot, run_dialog + events, tab_widget_count, qtbot: QtBot, run_dialog ): qtbot.waitUntil( lambda: run_dialog._tab_widget.count() == tab_widget_count, timeout=5000 @@ -615,7 +635,7 @@ def test_run_dialog_memory_usage_showing( ], ) def test_run_dialog_fm_label_show_correct_info( - events, event_queue, tab_widget_count, expected_host_info, qtbot: QtBot, run_dialog + events, tab_widget_count, expected_host_info, qtbot: QtBot, run_dialog ): qtbot.waitUntil( lambda: run_dialog._tab_widget.count() == tab_widget_count, timeout=5000 @@ -891,7 +911,7 @@ def dialog_appeared_and_test(): ], ) def test_forward_model_overview_label_selected_on_tab_change( - events, event_queue, tab_widget_count, qtbot: QtBot, run_dialog + events, tab_widget_count, qtbot: QtBot, run_dialog ): def qt_bot_click_realization(realization_index: int, iteration: int) -> None: view = run_dialog._tab_widget.widget(iteration)._real_view @@ -1033,7 +1053,7 @@ def test_that_file_dialog_close_when_run_dialog_hidden(qtbot: QtBot, run_dialog) ], ) def test_that_run_dialog_clears_warnings_when_rerun( - events, event_queue, qtbot, monkeypatch, run_dialog + events, qtbot, monkeypatch, run_dialog ): qtbot.wait_until(run_dialog.is_experiment_done, timeout=5000) assert len(run_dialog.post_experiment_warnings) > 0 @@ -1098,7 +1118,7 @@ def test_that_run_dialog_clears_warnings_when_rerun( ], ) def test_that_experiment_with_a_scheduler_warning_event_shows_a_warning_dialog( - events, event_queue, qtbot: QtBot, run_dialog: RunDialog + events, qtbot: QtBot, run_dialog: RunDialog ): ensemble_evaluation_warning_box = wait_for_child(run_dialog, qtbot, QMessageBox) @@ -1182,7 +1202,7 @@ def test_that_runpath_creation_events_add_update_and_remove_tab(qtbot: QtBot) -> @pytest.mark.usefixtures("use_tmpdir") -@pytest.mark.timeout(20) +@pytest.mark.timeout(60) @pytest.mark.slow def test_that_terminating_experiment_during_hooked_workflows_stops_run_dialog( qtbot: QtBot, monkeypatch @@ -1255,50 +1275,55 @@ def test_that_terminating_experiment_during_hooked_workflows_stops_run_dialog( args_mock = Mock() args_mock.config = config_file ert_config = ErtConfig.from_file(config_file) - gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") - qtbot.addWidget(gui) - experiment_panel = gui.findChild(ExperimentPanel) - assert experiment_panel - simulation_mode_combo = experiment_panel.findChild(QComboBox) - assert simulation_mode_combo - simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) - simulation_settings = gui.findChild(EnsembleExperimentPanel) - simulation_settings._experiment_name_field.setText("new_experiment_name") - run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") - assert run_experiment - qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) - qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=5000) - run_dialog = gui.findChild(RunDialog) - assert run_dialog - kill_button = run_dialog.kill_button + storage_path = Path("storage") + storage_path.mkdir() + with ErtServerController.init_service(project=storage_path.absolute()): + gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") + qtbot.addWidget(gui) + experiment_panel = gui.findChild(ExperimentPanel) + assert experiment_panel + simulation_mode_combo = experiment_panel.findChild(QComboBox) + assert simulation_mode_combo + simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) + simulation_settings = gui.findChild(EnsembleExperimentPanel) + simulation_settings._experiment_name_field.setText("new_experiment_name") + run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") + assert run_experiment + qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) + qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=30000) + run_dialog = gui.findChild(RunDialog) + assert run_dialog + kill_button = run_dialog.kill_button - qtbot.waitUntil(file_a.exists, timeout=10000) + qtbot.waitUntil(file_a.exists, timeout=30000) - def handle_dialog() -> None: - terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) - dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() - yes_button = next(button for button in dialog_buttons if "Yes" in button.text()) - qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) + def handle_dialog() -> None: + terminate_dialog = wait_for_child(run_dialog, qtbot, QMessageBox) + dialog_buttons = terminate_dialog.findChild(QDialogButtonBox).buttons() + yes_button = next( + button for button in dialog_buttons if "Yes" in button.text() + ) + qtbot.mouseClick(yes_button, Qt.MouseButton.LeftButton) - with qtbot.waitSignal(run_dialog.experiment_done, timeout=10000): - QTimer.singleShot(100, handle_dialog) - assert kill_button.isEnabled() - assert kill_button.text() == "Terminate experiment" - qtbot.mouseClick(kill_button, Qt.MouseButton.LeftButton) - assert not kill_button.isEnabled() - assert kill_button.text() == "Terminating" + with qtbot.waitSignal(run_dialog.experiment_done, timeout=30000): + QTimer.singleShot(100, handle_dialog) + assert kill_button.isEnabled() + assert kill_button.text() == "Terminate experiment" + qtbot.mouseClick(kill_button, Qt.MouseButton.LeftButton) + assert not kill_button.isEnabled() + assert kill_button.text() == "Terminating" - qtbot.waitUntil(lambda: run_dialog.fail_msg_box is not None, timeout=5000) + qtbot.waitUntil(lambda: run_dialog.fail_msg_box is not None, timeout=5000) - assert file_a.exists() - assert not file_b.exists() - assert not file_c.exists() - assert run_dialog.fail_msg_box is not None - assert kill_button.text() == "Terminate experiment" - assert not kill_button.isEnabled() - assert ( - "Experiment cancelled by user during workflows" - in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") - .findChild(QLabel) - .text() - ) + assert file_a.exists() + assert not file_b.exists() + assert not file_c.exists() + assert run_dialog.fail_msg_box is not None + assert kill_button.text() == "Terminate experiment" + assert not kill_button.isEnabled() + assert ( + "Experiment cancelled by user during workflows" + in run_dialog.fail_msg_box.findChild(QWidget, name="suggestor_messages") + .findChild(QLabel) + .text() + ) diff --git a/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py b/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py index 453e47543e1..683fd00a4bd 100644 --- a/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py +++ b/tests/ert/unit_tests/resources/test_run_eclipse_simulator.py @@ -8,7 +8,7 @@ import pytest from ert.plugins import ErtPluginManager -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_run_flow_simulator.py b/tests/ert/unit_tests/resources/test_run_flow_simulator.py index 35e233bd3a2..6cc4642bf89 100644 --- a/tests/ert/unit_tests/resources/test_run_flow_simulator.py +++ b/tests/ert/unit_tests/resources/test_run_flow_simulator.py @@ -6,7 +6,7 @@ import pytest -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py b/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py index 7bd5a3cff6e..a48a3272b1c 100644 --- a/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py +++ b/tests/ert/unit_tests/resources/test_run_reservoirsimulator.py @@ -10,7 +10,7 @@ import pytest import resfo -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_shell.py b/tests/ert/unit_tests/resources/test_shell.py index b72de6e2547..e873807343f 100644 --- a/tests/ert/unit_tests/resources/test_shell.py +++ b/tests/ert/unit_tests/resources/test_shell.py @@ -11,7 +11,7 @@ from ert.config import ErtConfig from ert.config.workflow_job import ExecutableWorkflow from ert.plugins import get_site_plugins -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/resources/test_templating.py b/tests/ert/unit_tests/resources/test_templating.py index 6ac8b3db2b1..c09b9ce0800 100644 --- a/tests/ert/unit_tests/resources/test_templating.py +++ b/tests/ert/unit_tests/resources/test_templating.py @@ -7,7 +7,7 @@ import jinja2 import pytest -from tests.ert.utils import SOURCE_DIR +from tests.conftest import SOURCE_DIR from ._import_from_location import import_from_location diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json index 5e707377626..b4eb910afba 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/heat_equationconfig.ert/config.json @@ -597,5 +597,6 @@ "shape_id": 3 } ], + "type": "ensemble_information_filter", "experiment_type": "Ensemble Information Filter" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json index f15485f8fcb..97a52ee5b89 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/poly_examplepoly.ert/poly.json @@ -304,5 +304,6 @@ "shape_id": null } ], + "type": "ensemble_information_filter", "experiment_type": "Ensemble Information Filter" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 2d5e362eb4b..5b44efb65df 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_enif_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -452,5 +452,6 @@ "shape_id": null } ], + "type": "ensemble_information_filter", "experiment_type": "Ensemble Information Filter" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json index f75dfb9a453..9554f5db754 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/heat_equationconfig.ert/config.json @@ -582,6 +582,7 @@ "shape_id": 3 } ], + "type": "ensemble_experiment", "target_ensemble": "the_experiment", "experiment_type": "Ensemble Experiment" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json index 211970ed822..ca7b9ee9ef7 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/poly_examplepoly.ert/poly.json @@ -289,6 +289,7 @@ "shape_id": null } ], + "type": "ensemble_experiment", "target_ensemble": "the_experiment", "experiment_type": "Ensemble Experiment" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 9c13ae3e182..8d0517f3787 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_experiment_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -437,6 +437,7 @@ "shape_id": null } ], + "type": "ensemble_experiment", "target_ensemble": "the_experiment", "experiment_type": "Ensemble Experiment" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json index 768fcf95883..a131a5574a2 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/heat_equationconfig.ert/config.json @@ -597,5 +597,6 @@ "shape_id": 3 } ], + "type": "ensemble_smoother", "experiment_type": "Ensemble Smoother" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json index 30225a99624..14cdbe9077f 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/poly_examplepoly.ert/poly.json @@ -304,5 +304,6 @@ "shape_id": null } ], + "type": "ensemble_smoother", "experiment_type": "Ensemble Smoother" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 52c39b2b2ea..fe05c49ed94 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_ensemble_smoother_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -452,5 +452,6 @@ "shape_id": null } ], + "type": "ensemble_smoother", "experiment_type": "Ensemble Smoother" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json index 5c53671d04c..2a0015093ab 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/heat_equationconfig.ert/config.json @@ -597,6 +597,7 @@ "shape_id": 3 } ], + "type": "multiple_data_assimilation", "restart_run": false, "prior_ensemble_id": null, "weights": "4, 2, 1", diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json index ba8827f04a0..84ce6128bdd 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/poly_examplepoly.ert/poly.json @@ -304,6 +304,7 @@ "shape_id": null } ], + "type": "multiple_data_assimilation", "restart_run": false, "prior_ensemble_id": null, "weights": "4, 2, 1", diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index b7c3a0a4e18..f0d6b7f6f14 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_esmda_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -452,6 +452,7 @@ "shape_id": null } ], + "type": "multiple_data_assimilation", "restart_run": false, "prior_ensemble_id": null, "weights": "4, 2, 1", diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json index 6e36d347f23..60dfb5ca3d6 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/heat_equationconfig.ert/config.json @@ -213,6 +213,7 @@ "random_seed": 1, "start_iteration": 0, "minimum_required_realizations": 3, + "type": "evaluate_ensemble", "ensemble_id": "00000000-0000-0000-0000-000000000000", "experiment_type": "Evaluate Ensemble" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json index 578854fdc33..c557fdec5c8 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/poly_examplepoly.ert/poly.json @@ -174,6 +174,7 @@ "random_seed": 1, "start_iteration": 0, "minimum_required_realizations": 1, + "type": "evaluate_ensemble", "ensemble_id": "00000000-0000-0000-0000-000000000000", "experiment_type": "Evaluate Ensemble" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index abfdd8b9a66..7fe4f201cfd 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_evaluate_ensemble_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -148,6 +148,7 @@ "random_seed": 1, "start_iteration": 0, "minimum_required_realizations": 3, + "type": "evaluate_ensemble", "ensemble_id": "00000000-0000-0000-0000-000000000000", "experiment_type": "Evaluate Ensemble" } diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json index e6f8faf5a80..e75f3002eba 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/heat_equationconfig.ert/config.json @@ -228,6 +228,7 @@ }, "auto_scale_observations": [] }, + "type": "manual_update", "ensemble_id": "00000000-0000-0000-0000-000000000000", "ert_templates": [], "experiment_type": "Manual Update" diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json index 81d74a18841..9b71d142bb4 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/poly_examplepoly.ert/poly.json @@ -189,6 +189,7 @@ }, "auto_scale_observations": [] }, + "type": "manual_update", "ensemble_id": "00000000-0000-0000-0000-000000000000", "ert_templates": [], "experiment_type": "Manual Update" diff --git a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json index 884f946bb17..5be3cd85679 100644 --- a/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json +++ b/tests/ert/unit_tests/run_models/snapshots/test_experiment_serialization/test_that_dumped_manual_update_matches_snapshot/snake_oilsnake_oil.ert/snake_oil.json @@ -163,6 +163,7 @@ }, "auto_scale_observations": [] }, + "type": "manual_update", "ensemble_id": "00000000-0000-0000-0000-000000000000", "ert_templates": [ [ diff --git a/tests/ert/unit_tests/run_models/test_model_factory.py b/tests/ert/unit_tests/run_models/test_model_factory.py index 9e4e218719f..d27e476a7ef 100644 --- a/tests/ert/unit_tests/run_models/test_model_factory.py +++ b/tests/ert/unit_tests/run_models/test_model_factory.py @@ -28,6 +28,7 @@ model_factory, ) from ert.run_models.model_factory import ( + _instantiate_run_model, _setup_ensemble_information_filter, _setup_ensemble_smoother, _setup_multiple_data_assimilation, @@ -123,7 +124,7 @@ def test_custom_realizations(): def test_setup_single_test_run(tmp_path): - model = model_factory._setup_single_test_run( + config = model_factory._setup_single_test_run( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( current_ensemble="current-ensemble", @@ -131,14 +132,14 @@ def test_setup_single_test_run(tmp_path): random_seed=None, experiment_name=None, ), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, SingleTestRun) assert model._storage.path == tmp_path def test_setup_single_test_run_with_ensemble(tmp_path): - model = model_factory._setup_single_test_run( + config = model_factory._setup_single_test_run( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( current_ensemble="current-ensemble", @@ -146,14 +147,14 @@ def test_setup_single_test_run_with_ensemble(tmp_path): random_seed=None, experiment_name=None, ), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, SingleTestRun) assert model._storage.path == tmp_path def test_setup_ensemble_experiment(tmp_path): - model = model_factory._setup_ensemble_experiment( + config = model_factory._setup_ensemble_experiment( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( realizations=None, @@ -162,8 +163,8 @@ def test_setup_ensemble_experiment(tmp_path): target_ensemble=None, experiment_name="ensemble_experiment", ), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, EnsembleExperiment) assert model.active_realizations == [True] * 100 @@ -171,7 +172,7 @@ def test_setup_ensemble_experiment(tmp_path): @pytest.mark.filterwarnings("ignore:MIN_REALIZATIONS") def test_setup_ensemble_smoother(tmp_path): - model = model_factory._setup_ensemble_smoother( + config = model_factory._setup_ensemble_smoother( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( realizations="0-4,7,8", @@ -180,8 +181,8 @@ def test_setup_ensemble_smoother(tmp_path): experiment_name="just_smoothing", ), ObservationSettings(), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, EnsembleSmoother) assert ( model.active_realizations @@ -191,7 +192,7 @@ def test_setup_ensemble_smoother(tmp_path): @pytest.mark.filterwarnings("ignore:MIN_REALIZATIONS") def test_setup_multiple_data_assimilation(tmp_path): - model = model_factory._setup_multiple_data_assimilation( + config = model_factory._setup_multiple_data_assimilation( ErtConfig.from_file_contents(f"NUM_REALIZATIONS 100\nENSPATH {tmp_path}"), Namespace( realizations="0-4,8", @@ -203,8 +204,8 @@ def test_setup_multiple_data_assimilation(tmp_path): starting_iteration=0, ), ObservationSettings(), - queue.SimpleQueue(), ) + model = _instantiate_run_model(config, queue.SimpleQueue()) assert isinstance(model, MultipleDataAssimilation) assert model.weights == "6,4,2" assert model._parsed_weights == MultipleDataAssimilation.parse_weights("6,4,2") @@ -269,9 +270,10 @@ def test_multiple_data_assimilation_restart_paths( with patch( "ert.run_models.run_model.Storage.get_ensemble", return_value=ensemble_mock ): - model = model_factory._setup_multiple_data_assimilation( - config, args, ObservationSettings(), queue.SimpleQueue() + config = model_factory._setup_multiple_data_assimilation( + config, args, ObservationSettings() ) + model = _instantiate_run_model(config, queue.SimpleQueue()) base_path = tmp_path / "simulations" expected_path = [str(base_path / expected) for expected in expected_path] assert set(model.paths) == set(expected_path) @@ -299,7 +301,7 @@ def test_num_realizations_specified_incorrectly_raises(analysis_mode): ConfigValidationError, match="Number of active realizations must be at least 2 for an update step", ): - analysis_mode(config, args, ObservationSettings(), queue.SimpleQueue()) + analysis_mode(config, args, ObservationSettings()) @pytest.mark.parametrize( @@ -372,4 +374,4 @@ def test_that_setting_up_experiment_with_update_step_raises_config_validation_er ConfigValidationError, match="Number of active realizations must be at least 2 for an update step", ): - experiment_setup_method(config, args, MagicMock(), MagicMock()) + experiment_setup_method(config, args, MagicMock()) diff --git a/tests/ert/utils.py b/tests/ert/utils.py index 0b2f642826a..14f15d9f74f 100644 --- a/tests/ert/utils.py +++ b/tests/ert/utils.py @@ -5,7 +5,6 @@ import time import uuid from collections.abc import Callable -from pathlib import Path from typing import TYPE_CHECKING, Self import zmq @@ -29,31 +28,6 @@ from ert.scheduler.driver import Driver -def source_dir() -> Path: - src = Path("@CMAKE_CURRENT_SOURCE_DIR@/../..") - if src.is_dir(): - return src.relative_to(Path.cwd()) - - # If the file was not correctly configured by cmake, look for the source - # folder, assuming the build folder is inside the source folder. - current_path = Path(__file__) - while current_path != Path("/"): - if (current_path / ".git").is_dir(): - return current_path - # This is to find root dir for git worktrees - elif (current_path / ".git").is_file(): - with (current_path / ".git").open(encoding="utf-8") as f: - for line in f: - if "gitdir:" in line: - return current_path - - current_path = current_path.parent - raise RuntimeError("Cannot find the source folder") - - -SOURCE_DIR: Path = source_dir() - - def wait_until(func, interval=0.5, timeout=30): """Waits until func returns True. diff --git a/tests/everest/conftest.py b/tests/everest/conftest.py index bd44a901c0a..3818193d599 100644 --- a/tests/everest/conftest.py +++ b/tests/everest/conftest.py @@ -14,6 +14,7 @@ import yaml from ert.base_model_context import use_runtime_plugins +from ert.config import ErtConfig from ert.config.queue_config import LocalQueueOptions, LsfQueueOptions from ert.ensemble_evaluator import EvaluatorServerConfig from ert.plugins import ErtRuntimePlugins, get_site_plugins @@ -172,6 +173,11 @@ def copy_math_func_test_data_to_tmp(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) +@pytest.fixture +def poly_ert_config(poly_case) -> ErtConfig: + return ErtConfig.from_file("poly.ert") + + @pytest.fixture def cached_example(pytestconfig): cache = pytestconfig.cache diff --git a/tests/everest/test_everserver.py b/tests/everest/test_everserver.py index 14925264d4f..d6cf422cb43 100644 --- a/tests/everest/test_everserver.py +++ b/tests/everest/test_everserver.py @@ -13,8 +13,14 @@ from ert.config import ConfigWarning from ert.dark_storage.app import app -from ert.dark_storage.endpoints.experiment_server import ExperimentRunnerState, _runs +from ert.dark_storage.endpoints.experiment_server import ( + ExperimentRunner, + ExperimentRunnerState, + _runs, +) from ert.ensemble_evaluator import EndEvent +from ert.namespace import Namespace +from ert.run_models.model_factory import build_run_model_config from ert.scheduler.event import FinishedEvent from ert.services import create_ertserver_client from ert.storage import ExperimentState @@ -30,8 +36,13 @@ from everest.strings import ( OPT_FAILURE_ALL_REALIZATIONS, OPT_FAILURE_REALIZATIONS, + EverEndpoints, ) +_FAKE_ENSEMBLE_ID = "00000000-0000-0000-0000-000000000001" + +_START_EXPERIMENT_URL = f"/experiment_server/{EverEndpoints.start_experiment}" + @pytest.fixture def setup_client(monkeypatch): @@ -377,3 +388,182 @@ def receive_event(): result.append(await receive_task) assert result == [jsonable_encoder(expected_result)] + + +@pytest.fixture +def start_experiment_client(monkeypatch): + original = dict(_runs) + monkeypatch.setenv("ERT_STORAGE_TOKEN", "password") + client = TestClient(app) + credentials = b64encode(b"username:password").decode() + yield client, {"Authorization": f"Basic {credentials}"} + _runs.clear() + _runs.update(original) + + +@pytest.mark.parametrize( + ("mode", "extra_kwargs"), + [ + pytest.param( + "test_run", + { + "experiment_name": None, + "current_ensemble": "prior", + "realizations": None, + }, + id="single_test_run", + ), + pytest.param( + "ensemble_experiment", + { + "experiment_name": "exp", + "current_ensemble": "prior", + "realizations": None, + }, + id="ensemble_experiment", + ), + pytest.param( + "ensemble_smoother", + { + "experiment_name": "exp", + "target_ensemble": "prior_%d", + "realizations": None, + }, + id="ensemble_smoother", + ), + pytest.param( + "ensemble_information_filter", + { + "experiment_name": "exp", + "target_ensemble": "prior_%d", + "realizations": None, + }, + id="ensemble_information_filter", + ), + pytest.param( + "es_mda", + { + "experiment_name": "exp", + "target_ensemble": "prior_%d", + "weights": "4, 2, 1", + "restart_ensemble_id": None, + "realizations": None, + }, + id="multiple_data_assimilation", + ), + pytest.param( + "manual_update", + { + "ensemble_id": _FAKE_ENSEMBLE_ID, + "target_ensemble": "prior_1", + "realizations": None, + }, + id="manual_update", + ), + pytest.param( + "evaluate_ensemble", + {"ensemble_id": _FAKE_ENSEMBLE_ID, "realizations": None}, + id="evaluate_ensemble", + ), + ], +) +def test_that_start_experiment_accepts_all_ert_run_model_config_types( + start_experiment_client, poly_ert_config, mode, extra_kwargs +): + client, auth_headers = start_experiment_client + run_model_config = build_run_model_config( + poly_ert_config, Namespace(mode=mode, **extra_kwargs) + ) + payload = run_model_config.model_dump(mode="json") + + with patch.object(ExperimentRunner, "run", new_callable=AsyncMock) as mock_run: + response = client.post( + _START_EXPERIMENT_URL, json=payload, headers=auth_headers + ) + mock_run.assert_called_once() + assert response.status_code == 200 + body = response.json() + assert "run_id" in body + assert body["run_id"] in _runs + + +def test_that_start_experiment_sets_paths_from_ert_config( + start_experiment_client, poly_ert_config +): + client, auth_headers = start_experiment_client + run_model_config = build_run_model_config( + poly_ert_config, + Namespace( + mode="ensemble_experiment", + experiment_name="exp", + current_ensemble="prior", + realizations=None, + ), + ) + payload = run_model_config.model_dump(mode="json") + + with patch.object(ExperimentRunner, "run", new_callable=AsyncMock): + response = client.post( + _START_EXPERIMENT_URL, json=payload, headers=auth_headers + ) + + assert response.status_code == 200 + run_id = response.json()["run_id"] + run_state = _runs[run_id] + assert str(run_state.config_path) == str(run_model_config.user_config_file) + assert run_state.run_path == run_model_config.runpath_config.runpath_format_string + assert run_state.storage_path == run_model_config.storage_path + assert run_state.start_time_unix is not None + + +def test_that_start_experiment_with_invalid_ert_payload_returns_an_error_response(): + client = TestClient(app, raise_server_exceptions=False) + credentials = b64encode(b"username:password").decode() + auth_headers = {"Authorization": f"Basic {credentials}"} + response = client.post( + _START_EXPERIMENT_URL, + json={"type": "not_a_valid_type"}, + headers=auth_headers, + ) + assert response.status_code >= 400 + + +def test_that_start_experiment_requires_authentication( + start_experiment_client, poly_ert_config +): + client, _ = start_experiment_client + run_model_config = build_run_model_config( + poly_ert_config, + Namespace( + mode="ensemble_experiment", + experiment_name="exp", + current_ensemble="prior", + realizations=None, + ), + ) + response = client.post( + _START_EXPERIMENT_URL, json=run_model_config.model_dump(mode="json") + ) + assert response.status_code == 401 + + +def test_that_start_experiment_rejects_wrong_password( + start_experiment_client, poly_ert_config +): + client, _ = start_experiment_client + wrong_credentials = b64encode(b"username:wrong_password").decode() + run_model_config = build_run_model_config( + poly_ert_config, + Namespace( + mode="ensemble_experiment", + experiment_name="exp", + current_ensemble="prior", + realizations=None, + ), + ) + response = client.post( + _START_EXPERIMENT_URL, + json=run_model_config.model_dump(mode="json"), + headers={"Authorization": f"Basic {wrong_credentials}"}, + ) + assert response.status_code == 401 diff --git a/tests/everest/test_templating.py b/tests/everest/test_templating.py index a70c4b80dac..47e852aa886 100644 --- a/tests/everest/test_templating.py +++ b/tests/everest/test_templating.py @@ -12,8 +12,8 @@ from ert.plugins import get_site_plugins from ert.run_models.everest_run_model import EverestRunModel from everest.config import EverestConfig, InstallTemplateConfig +from tests.conftest import SOURCE_DIR from tests.ert.unit_tests.resources._import_from_location import import_from_location -from tests.ert.utils import SOURCE_DIR from tests.everest.utils import everest_config_with_defaults CONFIG = { diff --git a/tests/everest/test_yaml_parser.py b/tests/everest/test_yaml_parser.py index 0a9f73ce91a..ff7c2b64325 100644 --- a/tests/everest/test_yaml_parser.py +++ b/tests/everest/test_yaml_parser.py @@ -24,7 +24,7 @@ def test_read_file(tmp_path, monkeypatch): encoding="utf-8", ) everest_config = EverestConfig.load_file("config.yml") - keys = ["config_path", "controls", "model", "objective_functions"] + keys = ["config_path", "controls", "model", "objective_functions", "type"] assert sorted(everest_config.to_dict().keys()) == sorted(keys) exp_dir, exp_fn = os.path.split(os.path.realpath("config.yml"))