From aede32e27230d15b860509e3fcbe8d5cc15b3eba Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Fri, 30 Jan 2026 16:20:00 -0500 Subject: [PATCH 1/7] getkf calculation in two steps --- src/swell/suites/localensembleda/flow.cylc | 32 ++- .../suites/localensembleda/suite_config.py | 47 ++-- src/swell/tasks/run_jedi_etkf_observer.py | 225 +++++++++++++++++ src/swell/tasks/run_jedi_etkf_solver.py | 226 ++++++++++++++++++ .../run_jedi_local_ensemble_da_executable.py | 17 +- .../tasks/run_jedi_variational_executable.py | 5 + src/swell/tasks/task_questions.py | 72 ++++++ src/swell/utilities/slurm.py | 2 + src/swell/utilities/yaml_utils.py | 19 ++ 9 files changed, 603 insertions(+), 42 deletions(-) create mode 100644 src/swell/tasks/run_jedi_etkf_observer.py create mode 100644 src/swell/tasks/run_jedi_etkf_solver.py create mode 100644 src/swell/utilities/yaml_utils.py diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index 8675bccba..3c7e6cefa 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -61,9 +61,10 @@ GetEnsembleGeosExperiment-{{model_component}} => sync_point - sync_point => RunJediObsfiltersExecutable-{{model_component}} {% if skip_ensemble_hofx %} - sync_point => RunJediObsfiltersExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + sync_point => RunJediObsfiltersExecutable-{{model_component}} + RunJediObsfiltersExecutable-{{model_component}} => RunJediEtkfObserver-{{model_component}} + RunJediEtkfObserver-{{model_component}} => RunJediEtkfSolver-{{model_component}} {% else %} # Run hofx for ensemble members according to strategy {% if ensemble_hofx_strategy == 'serial' %} @@ -86,17 +87,17 @@ # EvaIncrement - RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaIncrement-{{model_component}} + RunJediEtkfSolver-{{model_component}} => EvaIncrement-{{model_component}} # EvaObservations - # RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} + RunJediEtkfSolver-{{model_component}} => EvaObservations-{{model_component}} # Save observations - # RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + # RunJediEtkfSolver-{{model_component}} => SaveObsDiags-{{model_component}} # Clean up large files # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & - EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} + # EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} @@ -211,6 +212,24 @@ --{{key}} = {{value}} {%- endfor %} + [[RunJediEtkfObserver-{{model_component}}]] + script = "swell task RunJediEtkfObserver $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEtkfObserver"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEtkfObserver"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediEtkfSolver-{{model_component}}]] + script = "swell task RunJediEtkfSolver $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEtkfSolver"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEtkfSolver"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + [[EvaIncrement-{{model_component}}]] script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" @@ -232,7 +251,6 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} - [[sync_point]] script = true # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index 437dd35ab..22cb10360 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -39,21 +39,25 @@ class SuiteConfig(QuestionContainer, Enum): 'rtodling/archive/Restarts/JEDI/541x'), qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' 'rtodling/archive/541/Milan'), - qd.npx_proc(3), - qd.npy_proc(3), + qd.npx_proc(4), + qd.npy_proc(4), qd.cycle_times(['T00']), qd.ensemble_num_members(3), qd.skip_ensemble_hofx(True), qd.local_ensemble_solver("Deterministic GETKF"), - qd.local_ensemble_use_linear_observer(False), + qd.local_ensemble_use_linear_observer(True), qd.ensmean_only(False), qd.local_ensemble_save_posterior_mean(True), qd.local_ensemble_save_posterior_mean_increment(True), qd.local_ensemble_save_posterior_ensemble(False), qd.local_ensemble_save_posterior_ensemble_increments(False), - qd.obs_thinning_rej_fraction(0.75), + qd.obs_thinning_rej_fraction(0.9), qd.observations([ + "amsua_metop-b", "atms_n20", + "iasi_metop-b", + "mhs_n19", + "sondes", ]), qd.window_type("4D"), qd.clean_patterns(['*.txt']) @@ -78,11 +82,11 @@ class SuiteConfig(QuestionContainer, Enum): 'rtodling/archive/Restarts/JEDI/541x'), qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' 'rtodling/archive/541/Milan'), - qd.npx_proc(4), - qd.npy_proc(4), - # qd.perhost(32), + qd.npx_proc(8), + qd.npy_proc(8), + qd.perhost(96), qd.cycle_times(['T00']), - qd.ensemble_num_members(16), + qd.ensemble_num_members(32), qd.skip_ensemble_hofx(True), qd.local_ensemble_solver("Deterministic GETKF"), qd.local_ensemble_use_linear_observer(True), @@ -93,23 +97,25 @@ class SuiteConfig(QuestionContainer, Enum): qd.local_ensemble_save_posterior_ensemble_increments(False), qd.obs_thinning_rej_fraction(0.75), qd.observations([ - "aircraft_temperature", - "aircraft_wind", - "sondes", - "gps", + "airs_aqua", + "amsr2_gcom-w1", "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", "amsua_n15", "amsua_n18", "amsua_n19", - "amsr2_gcom-w1", "atms_n20", "atms_npp", "avhrr3_metop-b", "avhrr3_n18", "avhrr3_n19", - "scatwind", - "sfcship", - "sfc", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "gps", + "iasi_metop-b", + "iasi_metop-c", "mhs_metop-b", "mhs_metop-c", "mhs_n19", @@ -117,9 +123,12 @@ class SuiteConfig(QuestionContainer, Enum): "omi_aura", "ompsnm_npp", "pibal", - "ssmis_f17", - "amsua_metop-b", - "amsua_metop-c" + "satwind", + "scatwind", + "sfcship", + "sfc", + "sondes", + "ssmis_f17" ]), qd.window_type("4D"), qd.clean_patterns(['*.txt']) diff --git a/src/swell/tasks/run_jedi_etkf_observer.py b/src/swell/tasks/run_jedi_etkf_observer.py new file mode 100644 index 000000000..fbe648309 --- /dev/null +++ b/src/swell/tasks/run_jedi_etkf_observer.py @@ -0,0 +1,225 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import copy +import subprocess +from ruamel.yaml import YAML + +from swell.swell_path import get_swell_path +from swell.tasks.base.task_base import taskBase +from swell.utilities.yaml_utils import replace_key + +# -------------------------------------------------------------------------------------------------- + + +class RunJediEtkfObserver(taskBase): + + # ---------------------------------------------------------------------------------------------- + + def execute(self) -> None: + + # Jedi application name + # --------------------- + jedi_application = 'localensembleda' + + # Parse configuration + # ------------------- + window_type = self.config.window_type() + window_length = self.config.window_length() + background_time_offset = self.config.background_time_offset() + + jedi_forecast_model = self.config.jedi_forecast_model(None) + generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + + # Set the observing system records path + self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + + # Compute data assimilation window parameters + background_time = self.da_window_params.background_time(background_time_offset) + local_background_time = self.da_window_params.local_background_time(window_length, + window_type) + local_background_time_iso = self.da_window_params.local_background_time_iso(window_length, + window_type) + window_begin = self.da_window_params.window_begin(window_length) + window_begin_iso = self.da_window_params.window_begin_iso(window_length) + window_end_iso = self.da_window_params.window_end_iso(window_length) + + # Populate jedi interface templates dictionary + # -------------------------------------------- + self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) + self.jedi_rendering.add_key('window_length', window_length) + self.jedi_rendering.add_key('window_end_iso', window_end_iso) + + # Background + self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('local_background_time', local_background_time) + self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) + self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members()) + + # Geometry + self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) + self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) + self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + + # Observations + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('window_begin', window_begin) + + # Ensemble Localizations + self.jedi_rendering.add_key('horizontal_localization_method', + self.config.horizontal_localization_method()) + self.jedi_rendering.add_key('horizontal_localization_lengthscale', + self.config.horizontal_localization_lengthscale()) + self.jedi_rendering.add_key('horizontal_localization_max_nobs', + self.config.horizontal_localization_max_nobs()) + self.jedi_rendering.add_key('vertical_localization_method', + self.config.vertical_localization_method()) + self.jedi_rendering.add_key('vertical_localization_apply_log_transform', + self.config.vertical_localization_apply_log_transform()) + self.jedi_rendering.add_key('vertical_localization_lengthscale', + self.config.vertical_localization_lengthscale()) + self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord', + self.config.vertical_localization_ioda_vertical_coord()) + self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord_group', + self.config.vertical_localization_ioda_vertical_coord_group()) + self.jedi_rendering.add_key('vertical_localization_function', + self.config.vertical_localization_function()) + + # Driver + self.jedi_rendering.add_key('local_ensemble_solver', self.config.local_ensemble_solver()) + self.jedi_rendering.add_key('local_ensemble_inflation_rtps', + self.config.local_ensemble_inflation_rtps()) + self.jedi_rendering.add_key('local_ensemble_inflation_rtpp', + self.config.local_ensemble_inflation_rtpp()) + self.jedi_rendering.add_key('local_ensemble_inflation_mult', + self.config.local_ensemble_inflation_mult()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_mean', + self.config.local_ensemble_save_posterior_mean()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble', + self.config.local_ensemble_save_posterior_ensemble()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_mean_increment', + self.config.local_ensemble_save_posterior_mean_increment()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble_increments', + self.config.local_ensemble_save_posterior_ensemble_increments()) + self.jedi_rendering.add_key('local_ensemble_use_linear_observer', + self.config.local_ensemble_use_linear_observer()) + self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.skip_ensemble_hofx()) + + # Prevent both 'local_ensemble_save_posterior_mean' and + # 'local_ensemble_save_posterior_ensemble' from being true + # -------------------------------------------------------- + if self.config.local_ensemble_save_posterior_mean() and \ + self.config.local_ensemble_save_posterior_ensemble(): + raise ValueError("'local_ensemble_save_posterior_mean' and\ + 'local_ensemble_save_posterior_ensemble' cannot be both true!") + + # Open the JEDI config file and fill initial templates + # ---------------------------------------------------- + jedi_config_dict = self.jedi_rendering.render_oops_file('LocalEnsembleDA', + window_type, + jedi_forecast_model) + + # Assemble localizations + # ---------------------- + # # Vertical localizations have bug(s) - Commented out for now... + # vertLoc = {'localization method': self.config.vertical_localization_method(), + # 'apply log transformation': + # self.config.vertical_localization_apply_log_transform(), + # 'vertical lengthscale': self.config.vertical_localization_lengthscale(), + # 'ioda vertical coordinate': + # self.config.vertical_localization_ioda_vertical_coord(), + # 'ioda vertical coordinate group': + # self.config.vertical_localization_ioda_vertical_coord_group(), + # 'localization function': self.config.vertical_localization_function()} + # localizations = [horizLoc, vertLoc] if len(vertLoc) != 0 else [horizLoc] + + # Include ensemble localizations and halo types with each observation + # ------------------------------------------------------------------- + + swell_path = get_swell_path() + localization_path = os.path.join(swell_path, + f'configuration/jedi/interfaces/geos_atmosphere' + f'/observations/localization') + yaml = YAML() + # update localizations in dict + for observer in jedi_config_dict['observations']['observers']: + # Get observation name + observation_name = observer['observation_name'] + config_file = os.path.join(localization_path, f'{observation_name}.yaml') + with open(config_file, 'r') as f: + loc_list = yaml.load(f) + horizLoc = loc_list['obs localizations'] + localization = [horizLoc] + observer.update({'obs localizations': localization}) + observer['obs space'].update( + {'distribution': {'name': 'Halo', 'halo size': 5000.e3}}) + + # change variational bc to static bc + # ------------------------------------------------------------------- + change_vbc_to_sbc = False + if change_vbc_to_sbc: + for observer in jedi_config_dict['observations']['observers']: + if 'obs bias' in observer: + observer['obs bias'] = replace_key(observer['obs bias'], + "variational bc", "static bc") + model_component_meta = self.jedi_rendering.render_interface_meta() + jedi_executable = model_component_meta['executables'][f'{jedi_application}'] + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', + 'build', 'bin', jedi_executable) + + # seperate each obs and write to disk + # ------------------------------------------------------------------- + driver = jedi_config_dict['driver'] + driver['run as observer only'] = True + driver['read HX from disk'] = False + print(f'driver= {driver}') + + observers = jedi_config_dict + ["observations"]["observers"] + npx = 1 + npy = 2 + np = 6 * npx * npy + cmd = """ + export SLURM_MPI_TYPE=pmi2 + export I_MPI_PMI_LIBRARY=/usr/lib64/libpmi2.so + """ + cmd += f"cd {self.cycle_dir()} \n" + cmd += f"rm -f log.* logfile* \n" + for i, obs in enumerate(observers): + x0 = copy.deepcopy(jedi_config_dict) + x0["observations"]["observers"] = [obs] + x0['geometry']['layout'] = [npx, npy] + observation_name = obs['observation_name'] + tmp_file1 = os.path.join(self.cycle_dir(), f'diag_{observation_name}.yaml') + tmp_file2 = os.path.join(self.cycle_dir(), f'log.diag_{observation_name}') + with open(tmp_file1, "w") as f: + yaml.dump(x0, f) + cmd += ( + f"srun --exclusive --mpi=pmi2 -n {np} " + f"{jedi_executable_path} {tmp_file1} {tmp_file2} &\n" + ) + cmd += f"wait \n" + print(f'nobs = {i+1}') + np_use = (i+1) * np + np_total = eval(str(model_component_meta['total_processors'])) + error_msg = f'{i+1} obs: each {np} cores, np_use: {np_use} vs np_avail: {np_total}' + assert np_use <= np_total, error_msg + + if not generate_yaml_and_exit: + subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, check=True) + else: + print(f'intended mpi_command = {cmd}') + self.logger.info('YAML generated, now exiting.') + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_etkf_solver.py b/src/swell/tasks/run_jedi_etkf_solver.py new file mode 100644 index 000000000..eff0b1aac --- /dev/null +++ b/src/swell/tasks/run_jedi_etkf_solver.py @@ -0,0 +1,226 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +from ruamel.yaml import YAML + +from swell.swell_path import get_swell_path +from swell.tasks.base.task_base import taskBase +from swell.utilities.run_jedi_executables import run_executable +from swell.utilities.yaml_utils import replace_key + +# -------------------------------------------------------------------------------------------------- + + +class RunJediEtkfSolver(taskBase): + + # ---------------------------------------------------------------------------------------------- + + def execute(self) -> None: + + # Jedi application name + # --------------------- + jedi_application = 'localensembleda' + + # Parse configuration + # ------------------- + window_type = self.config.window_type() + window_length = self.config.window_length() + background_time_offset = self.config.background_time_offset() + + jedi_forecast_model = self.config.jedi_forecast_model(None) + generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + + # Set the observing system records path + self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + + # Compute data assimilation window parameters + background_time = self.da_window_params.background_time(background_time_offset) + local_background_time = self.da_window_params.local_background_time(window_length, + window_type) + local_background_time_iso = self.da_window_params.local_background_time_iso(window_length, + window_type) + window_begin = self.da_window_params.window_begin(window_length) + window_begin_iso = self.da_window_params.window_begin_iso(window_length) + window_end_iso = self.da_window_params.window_end_iso(window_length) + + # Populate jedi interface templates dictionary + # -------------------------------------------- + self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) + self.jedi_rendering.add_key('window_length', window_length) + self.jedi_rendering.add_key('window_end_iso', window_end_iso) + + # Background + self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('local_background_time', local_background_time) + self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) + self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members()) + + # Geometry + self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) + self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) + self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + + # Observations + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('window_begin', window_begin) + + # Ensemble Localizations + self.jedi_rendering.add_key('horizontal_localization_method', + self.config.horizontal_localization_method()) + self.jedi_rendering.add_key('horizontal_localization_lengthscale', + self.config.horizontal_localization_lengthscale()) + self.jedi_rendering.add_key('horizontal_localization_max_nobs', + self.config.horizontal_localization_max_nobs()) + self.jedi_rendering.add_key('vertical_localization_method', + self.config.vertical_localization_method()) + self.jedi_rendering.add_key('vertical_localization_apply_log_transform', + self.config.vertical_localization_apply_log_transform()) + self.jedi_rendering.add_key('vertical_localization_lengthscale', + self.config.vertical_localization_lengthscale()) + self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord', + self.config.vertical_localization_ioda_vertical_coord()) + self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord_group', + self.config.vertical_localization_ioda_vertical_coord_group()) + self.jedi_rendering.add_key('vertical_localization_function', + self.config.vertical_localization_function()) + + # Driver + self.jedi_rendering.add_key('local_ensemble_solver', self.config.local_ensemble_solver()) + self.jedi_rendering.add_key('local_ensemble_inflation_rtps', + self.config.local_ensemble_inflation_rtps()) + self.jedi_rendering.add_key('local_ensemble_inflation_rtpp', + self.config.local_ensemble_inflation_rtpp()) + self.jedi_rendering.add_key('local_ensemble_inflation_mult', + self.config.local_ensemble_inflation_mult()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_mean', + self.config.local_ensemble_save_posterior_mean()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble', + self.config.local_ensemble_save_posterior_ensemble()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_mean_increment', + self.config.local_ensemble_save_posterior_mean_increment()) + self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble_increments', + self.config.local_ensemble_save_posterior_ensemble_increments()) + self.jedi_rendering.add_key('local_ensemble_use_linear_observer', + self.config.local_ensemble_use_linear_observer()) + self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.skip_ensemble_hofx()) + + # Prevent both 'local_ensemble_save_posterior_mean' and + # 'local_ensemble_save_posterior_ensemble' from being true + # -------------------------------------------------------- + if self.config.local_ensemble_save_posterior_mean() and \ + self.config.local_ensemble_save_posterior_ensemble(): + raise ValueError("'local_ensemble_save_posterior_mean' and\ + 'local_ensemble_save_posterior_ensemble' cannot be both true!") + + # Jedi configuration file + # ----------------------- + jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml') + + # Output log file + # --------------- + output_log_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_log.log') + + # Open the JEDI config file and fill initial templates + # ---------------------------------------------------- + jedi_config_dict = self.jedi_rendering.render_oops_file('LocalEnsembleDA', + window_type, + jedi_forecast_model) + + # Assemble localizations + # ---------------------- + # # Vertical localizations have bug(s) - Commented out for now... + # vertLoc = {'localization method': self.config.vertical_localization_method(), + # 'apply log transformation': + # self.config.vertical_localization_apply_log_transform(), + # 'vertical lengthscale': self.config.vertical_localization_lengthscale(), + # 'ioda vertical coordinate': + # self.config.vertical_localization_ioda_vertical_coord(), + # 'ioda vertical coordinate group': + # self.config.vertical_localization_ioda_vertical_coord_group(), + # 'localization function': self.config.vertical_localization_function()} + # localizations = [horizLoc, vertLoc] if len(vertLoc) != 0 else [horizLoc] + + # Include ensemble localizations and halo types with each observation + # ------------------------------------------------------------------- + + swell_path = get_swell_path() + localization_path = os.path.join(swell_path, + f'configuration/jedi/interfaces/geos_atmosphere' + f'/observations/localization') + yaml = YAML() + # update localizations in dict + for observer in jedi_config_dict['observations']['observers']: + # Get observation name + observation_name = observer['observation_name'] + config_file = os.path.join(localization_path, f'{observation_name}.yaml') + with open(config_file, 'r') as f: + loc_list = yaml.load(f) + horizLoc = loc_list['obs localizations'] + localization = [horizLoc] + observer.update({'obs localizations': localization}) + observer['obs space'].update( + {'distribution': {'name': 'Halo', 'halo size': 5000.e3}}) + + # change variational bc to static bc + # ------------------------------------------------------------------- + change_vbc_to_sbc = False + if change_vbc_to_sbc: + for observer in jedi_config_dict['observations']['observers']: + if 'obs bias' in observer: + observer['obs bias'] = replace_key(observer['obs bias'], + "variational bc", "static bc") + + driver = jedi_config_dict['driver'] + driver['read HX from disk'] = True + driver['run as observer only'] = False + print(f'driver= {driver}') + + observers = jedi_config_dict["observations"]["observers"] + for i, obs in enumerate(observers): + observation_name = obs['observation_name'] + obs_file_read = obs['obs space']['obsdataout']['engine']['obsfile'] + print(f'\n obs_file_read = {obs_file_read}') + obs['obs space']['obsdatain']['engine']['obsfile'] = obs_file_read + dir_path = os.path.dirname(obs_file_read) + file_name = os.path.basename(obs_file_read) + obs['obs space']['obsdataout']['engine']['obsfile'] = os.path.join + (dir_path, 'solver.' + file_name) + + # bypass the writing of HofXs + # --------------------------- + bypass_HofXs = True + if bypass_HofXs: + for observer in jedi_config_dict['observations']['observers']: + del observer['obs space']['obsdataout'] + + with open(jedi_config_file, 'w') as f: + yaml.dump(jedi_config_dict, f) + + model_component_meta = self.jedi_rendering.render_interface_meta() + jedi_executable = model_component_meta['executables'][f'{jedi_application}'] + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', + jedi_executable) + np = eval(str(model_component_meta['total_processors'])) + perhost = self.config.perhost() + if not generate_yaml_and_exit: + run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, + jedi_config_file, output_log_file, perhost=perhost) + else: + mpi_command = "mpirun" + if not (perhost is None or perhost == "None"): + mpi_command += f" -perhost {perhost}" + mpi_command += f" -np {np} {jedi_executable_path} {jedi_config_file} {output_log_file}" + print(f'intended mpi_command = {mpi_command}') + self.logger.info('YAML generated, now exiting.') + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index b0d67a5b8..461bc50d6 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -13,27 +13,12 @@ from swell.swell_path import get_swell_path from swell.tasks.base.task_base import taskBase +from swell.utilities.yaml_utils import replace_key from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- -def replace_key(obj, old_key, new_key): - """ - Recursively replace dictionary keys in nested dictionaries/lists. - """ - if isinstance(obj, dict): - new_dict = {} - for k, v in obj.items(): - new_k = new_key if k == old_key else k - new_dict[new_k] = replace_key(v, old_key, new_key) - return new_dict - elif isinstance(obj, list): - return [replace_key(item, old_key, new_key) for item in obj] - else: - return obj - - class RunJediLocalEnsembleDaExecutable(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index dc634cbd7..40e5098ec 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -147,6 +147,11 @@ def execute(self) -> None: run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, output_log_file, perhost) else: + mpi_command = "mpirun" + if not (perhost is None or perhost == "None"): + mpi_command += f" -perhost {perhost}" + mpi_command += f" -np {np} {jedi_executable_path} {jedi_config_file} {output_log_file}" + print(f'intended mpi_command = {mpi_command}') self.logger.info('YAML generated, now exiting.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 974b15971..4efbdf6a4 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -672,6 +672,78 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + RunJediEtkfObserver = QuestionList( + list_name="RunJediEtkfObserver", + questions=[ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.horizontal_localization_lengthscale(), + qd.horizontal_localization_max_nobs(), + qd.horizontal_localization_method(), + qd.jedi_forecast_model(), + qd.local_ensemble_inflation_mult(), + qd.local_ensemble_inflation_rtpp(), + qd.local_ensemble_inflation_rtps(), + qd.local_ensemble_save_posterior_ensemble(), + qd.local_ensemble_save_posterior_ensemble_increments(), + qd.local_ensemble_save_posterior_mean(), + qd.local_ensemble_save_posterior_mean_increment(), + qd.local_ensemble_solver(), + qd.local_ensemble_use_linear_observer(), + qd.skip_ensemble_hofx(), + qd.total_processors(), + qd.vertical_localization_apply_log_transform(), + qd.vertical_localization_function(), + qd.vertical_localization_ioda_vertical_coord(), + qd.vertical_localization_ioda_vertical_coord_group(), + qd.vertical_localization_lengthscale(), + qd.vertical_localization_method(), + qd.perhost(), + qd.comparison_log_type('localensembleda'), + ] + ) + + # -------------------------------------------------------------------------------------------------- + + RunJediEtkfSolver = QuestionList( + list_name="RunJediEtkfSolver", + questions=[ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.horizontal_localization_lengthscale(), + qd.horizontal_localization_max_nobs(), + qd.horizontal_localization_method(), + qd.jedi_forecast_model(), + qd.local_ensemble_inflation_mult(), + qd.local_ensemble_inflation_rtpp(), + qd.local_ensemble_inflation_rtps(), + qd.local_ensemble_save_posterior_ensemble(), + qd.local_ensemble_save_posterior_ensemble_increments(), + qd.local_ensemble_save_posterior_mean(), + qd.local_ensemble_save_posterior_mean_increment(), + qd.local_ensemble_solver(), + qd.local_ensemble_use_linear_observer(), + qd.skip_ensemble_hofx(), + qd.total_processors(), + qd.vertical_localization_apply_log_transform(), + qd.vertical_localization_function(), + qd.vertical_localization_ioda_vertical_coord(), + qd.vertical_localization_ioda_vertical_coord_group(), + qd.vertical_localization_lengthscale(), + qd.vertical_localization_method(), + qd.perhost(), + qd.comparison_log_type('localensembleda'), + ] + ) + + # -------------------------------------------------------------------------------------------------- + RunJediUfoTestsExecutable = QuestionList( list_name="RunJediUfoTestsExecutable", questions=[ diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index c6ef1895d..9ca3fb165 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -83,6 +83,8 @@ def prepare_scheduling_dict( 'RunJediHofxEnsembleExecutable', 'RunJediHofxExecutable', 'RunJediLocalEnsembleDaExecutable', + 'RunJediEtkfObserver', + 'RunJediEtkfSolver', 'RunJediObsfiltersExecutable', 'RunJediUfoTestsExecutable', 'RunJediVariationalExecutable', diff --git a/src/swell/utilities/yaml_utils.py b/src/swell/utilities/yaml_utils.py new file mode 100644 index 000000000..fc73d28e8 --- /dev/null +++ b/src/swell/utilities/yaml_utils.py @@ -0,0 +1,19 @@ + +# -------------------------------------------------------------------------------------------------- + +def replace_key(obj, old_key, new_key): + """ + Recursively replace dictionary keys in nested dictionaries/lists. + """ + if isinstance(obj, dict): + new_dict = {} + for k, v in obj.items(): + new_k = new_key if k == old_key else k + new_dict[new_k] = replace_key(v, old_key, new_key) + return new_dict + elif isinstance(obj, list): + return [replace_key(item, old_key, new_key) for item in obj] + else: + return obj + +# -------------------------------------------------------------------------------------------------- From 6ea6a0d7215925c03bfcbe0f97a590483bc711f4 Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Mon, 2 Feb 2026 16:38:53 -0500 Subject: [PATCH 2/7] add task_default for slurm --- src/swell/tasks/run_jedi_etkf_observer.py | 3 +-- src/swell/utilities/slurm.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/swell/tasks/run_jedi_etkf_observer.py b/src/swell/tasks/run_jedi_etkf_observer.py index fbe648309..be8c040b4 100644 --- a/src/swell/tasks/run_jedi_etkf_observer.py +++ b/src/swell/tasks/run_jedi_etkf_observer.py @@ -184,8 +184,7 @@ def execute(self) -> None: driver['read HX from disk'] = False print(f'driver= {driver}') - observers = jedi_config_dict - ["observations"]["observers"] + observers = jedi_config_dict["observations"]["observers"] npx = 1 npy = 2 np = 6 * npx * npy diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 9ca3fb165..20be00299 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -46,7 +46,9 @@ def prepare_scheduling_dict( task_defaults = { "RunJediVariationalExecutable": {"all": {"nodes": 3}}, "RunJediUfoTestsExecutable": {"all": {"ntasks-per-node": 1}}, - "RunJediConvertStateSoca2ciceExecutable": {"all": {"nodes": 1}} + "RunJediConvertStateSoca2ciceExecutable": {"all": {"nodes": 1}}, + "RunJediEtkfObserver": {"all": {"nodes": 4}}, + "RunJediEtkfSolver": {"all": {"nodes": 4}} } # Global SLURM settings stored in $HOME/.swell/swell-slurm.yaml From 0a09ea219eb085d2875b5636333d502fbaa4f5a9 Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Tue, 3 Feb 2026 16:34:27 -0500 Subject: [PATCH 3/7] add flag to change varbc to static bc --- src/swell/suites/localensembleda/suite_config.py | 2 ++ src/swell/tasks/run_jedi_etkf_observer.py | 2 +- src/swell/tasks/run_jedi_etkf_solver.py | 1 + .../tasks/run_jedi_local_ensemble_da_executable.py | 10 ++++++---- src/swell/tasks/task_questions.py | 3 +++ src/swell/utilities/question_defaults.py | 13 +++++++++++++ 6 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index 22cb10360..c668da629 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -59,6 +59,7 @@ class SuiteConfig(QuestionContainer, Enum): "mhs_n19", "sondes", ]), + qd.change_vbc_to_sbc(False), qd.window_type("4D"), qd.clean_patterns(['*.txt']) ] @@ -130,6 +131,7 @@ class SuiteConfig(QuestionContainer, Enum): "sondes", "ssmis_f17" ]), + qd.change_vbc_to_sbc(False), qd.window_type("4D"), qd.clean_patterns(['*.txt']) ] diff --git a/src/swell/tasks/run_jedi_etkf_observer.py b/src/swell/tasks/run_jedi_etkf_observer.py index be8c040b4..766a57794 100644 --- a/src/swell/tasks/run_jedi_etkf_observer.py +++ b/src/swell/tasks/run_jedi_etkf_observer.py @@ -38,6 +38,7 @@ def execute(self) -> None: jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + change_vbc_to_sbc = self.config.change_vbc_to_sbc(False) # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) @@ -166,7 +167,6 @@ def execute(self) -> None: # change variational bc to static bc # ------------------------------------------------------------------- - change_vbc_to_sbc = False if change_vbc_to_sbc: for observer in jedi_config_dict['observations']['observers']: if 'obs bias' in observer: diff --git a/src/swell/tasks/run_jedi_etkf_solver.py b/src/swell/tasks/run_jedi_etkf_solver.py index eff0b1aac..cd7711036 100644 --- a/src/swell/tasks/run_jedi_etkf_solver.py +++ b/src/swell/tasks/run_jedi_etkf_solver.py @@ -34,6 +34,7 @@ def execute(self) -> None: window_type = self.config.window_type() window_length = self.config.window_length() background_time_offset = self.config.background_time_offset() + change_vbc_to_sbc = self.config.change_vbc_to_sbc(False) jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 461bc50d6..e01ead301 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -40,6 +40,7 @@ def execute(self) -> None: generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) ensmean_only = self.config.ensmean_only() ensmeanvariance_only = self.config.ensmeanvariance_only() + change_vbc_to_sbc = self.config.change_vbc_to_sbc(False) # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) @@ -193,10 +194,11 @@ def execute(self) -> None: # change variational bc to static bc # ------------------------------------------------------------------- - for observer in jedi_config_dict['observations']['observers']: - if 'obs bias' in observer: - observer['obs bias'] = replace_key(observer['obs bias'], - "variational bc", "static bc") + if change_vbc_to_sbc: + for observer in jedi_config_dict['observations']['observers']: + if 'obs bias' in observer: + observer['obs bias'] = replace_key(observer['obs bias'], + "variational bc", "static bc") # Write the expanded dictionary to YAML file (in rt mode) # ------------------------------------------ diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 4efbdf6a4..ce13c8db6 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -648,6 +648,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_lengthscale(), qd.vertical_localization_method(), qd.perhost(), + qd.change_vbc_to_sbc(), qd.comparison_log_type('localensembleda'), ] ) @@ -702,6 +703,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_lengthscale(), qd.vertical_localization_method(), qd.perhost(), + qd.change_vbc_to_sbc(), qd.comparison_log_type('localensembleda'), ] ) @@ -738,6 +740,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_lengthscale(), qd.vertical_localization_method(), qd.perhost(), + qd.change_vbc_to_sbc(), qd.comparison_log_type('localensembleda'), ] ) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index df01ae2cf..9f6bbb5b2 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -932,6 +932,19 @@ class local_ensemble_use_linear_observer(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class change_vbc_to_sbc(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "change_vbc_to_sbc" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Shall variational bc be changed to static bc in local ensemble DA yaml?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + @dataclass class minimizer(TaskQuestion): default_value: str = "defer_to_model" From dc00e707a82f73c59ef694341166778e88311378 Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Fri, 13 Feb 2026 16:53:22 -0500 Subject: [PATCH 4/7] update --- .../suites/localensembleda/suite_config.py | 4 ++-- src/swell/tasks/run_jedi_etkf_observer.py | 8 ++++---- src/swell/tasks/run_jedi_etkf_solver.py | 19 +++++-------------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index c668da629..a94dc3331 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -98,18 +98,18 @@ class SuiteConfig(QuestionContainer, Enum): qd.local_ensemble_save_posterior_ensemble_increments(False), qd.obs_thinning_rej_fraction(0.75), qd.observations([ + "aircraft_temperature", + "aircraft_wind", "airs_aqua", "amsr2_gcom-w1", "amsua_aqua", "amsua_metop-b", "amsua_metop-c", "amsua_n15", - "amsua_n18", "amsua_n19", "atms_n20", "atms_npp", "avhrr3_metop-b", - "avhrr3_n18", "avhrr3_n19", "cris-fsr_n20", "cris-fsr_npp", diff --git a/src/swell/tasks/run_jedi_etkf_observer.py b/src/swell/tasks/run_jedi_etkf_observer.py index 766a57794..4b80ed3ec 100644 --- a/src/swell/tasks/run_jedi_etkf_observer.py +++ b/src/swell/tasks/run_jedi_etkf_observer.py @@ -163,7 +163,7 @@ def execute(self) -> None: localization = [horizLoc] observer.update({'obs localizations': localization}) observer['obs space'].update( - {'distribution': {'name': 'Halo', 'halo size': 5000.e3}}) + {'distribution': {'name': 'RoundRobin', 'halo size': 1500.e3}}) # change variational bc to static bc # ------------------------------------------------------------------- @@ -186,7 +186,7 @@ def execute(self) -> None: observers = jedi_config_dict["observations"]["observers"] npx = 1 - npy = 2 + npy = 1 np = 6 * npx * npy cmd = """ export SLURM_MPI_TYPE=pmi2 @@ -215,8 +215,8 @@ def execute(self) -> None: assert np_use <= np_total, error_msg if not generate_yaml_and_exit: - subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, check=True) + subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=True) else: print(f'intended mpi_command = {cmd}') self.logger.info('YAML generated, now exiting.') diff --git a/src/swell/tasks/run_jedi_etkf_solver.py b/src/swell/tasks/run_jedi_etkf_solver.py index cd7711036..ba43a9d24 100644 --- a/src/swell/tasks/run_jedi_etkf_solver.py +++ b/src/swell/tasks/run_jedi_etkf_solver.py @@ -125,11 +125,11 @@ def execute(self) -> None: # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml') + jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_etkf_solver_config.yaml') # Output log file # --------------- - output_log_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_log.log') + output_log_file = os.path.join(self.cycle_dir(), f'jedi_etkf_solver_log.log') # Open the JEDI config file and fill initial templates # ---------------------------------------------------- @@ -170,11 +170,10 @@ def execute(self) -> None: localization = [horizLoc] observer.update({'obs localizations': localization}) observer['obs space'].update( - {'distribution': {'name': 'Halo', 'halo size': 5000.e3}}) + {'distribution': {'name': 'Halo', 'halo size': 1500.e3}}) # change variational bc to static bc # ------------------------------------------------------------------- - change_vbc_to_sbc = False if change_vbc_to_sbc: for observer in jedi_config_dict['observations']['observers']: if 'obs bias' in observer: @@ -194,15 +193,7 @@ def execute(self) -> None: obs['obs space']['obsdatain']['engine']['obsfile'] = obs_file_read dir_path = os.path.dirname(obs_file_read) file_name = os.path.basename(obs_file_read) - obs['obs space']['obsdataout']['engine']['obsfile'] = os.path.join - (dir_path, 'solver.' + file_name) - - # bypass the writing of HofXs - # --------------------------- - bypass_HofXs = True - if bypass_HofXs: - for observer in jedi_config_dict['observations']['observers']: - del observer['obs space']['obsdataout'] + obs['obs space']['obsdataout']['engine']['obsfile'] = os.path.join(dir_path, 'solver.' + file_name) with open(jedi_config_file, 'w') as f: yaml.dump(jedi_config_dict, f) @@ -212,7 +203,7 @@ def execute(self) -> None: jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) np = eval(str(model_component_meta['total_processors'])) - perhost = self.config.perhost() + perhost = self.config.perhost(None) if not generate_yaml_and_exit: run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, output_log_file, perhost=perhost) From 80b559b0b5f4eeb80171b1ba3aeb105b8ef97054 Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Mon, 9 Mar 2026 16:45:17 -0400 Subject: [PATCH 5/7] fix a code style issue --- src/swell/tasks/run_jedi_etkf_solver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/run_jedi_etkf_solver.py b/src/swell/tasks/run_jedi_etkf_solver.py index ba43a9d24..0e7bf6245 100644 --- a/src/swell/tasks/run_jedi_etkf_solver.py +++ b/src/swell/tasks/run_jedi_etkf_solver.py @@ -193,7 +193,9 @@ def execute(self) -> None: obs['obs space']['obsdatain']['engine']['obsfile'] = obs_file_read dir_path = os.path.dirname(obs_file_read) file_name = os.path.basename(obs_file_read) - obs['obs space']['obsdataout']['engine']['obsfile'] = os.path.join(dir_path, 'solver.' + file_name) + obs['obs space']['obsdataout']['engine']['obsfile'] = ( + os.path.join(dir_path, 'solver.' + file_name) + ) with open(jedi_config_file, 'w') as f: yaml.dump(jedi_config_dict, f) From bd50e75b41c5cbd5f9befb6af7727bd272c865cc Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Thu, 6 Aug 2026 10:27:04 -0400 Subject: [PATCH 6/7] Vertical localization parameters in ETKF (L2) (#729) * LocalEnsembleDA: * add keywords on vertical localization * correct a few typos --- .../geos_atmosphere/model/ensemble_solver.py | 9 ++++-- .../geos_atmosphere/task_questions.yaml | 11 ++++++- .../suites/localensembleda/suite_config.py | 6 ++++ src/swell/tasks/run_jedi_etkf_observer.py | 4 +++ src/swell/tasks/run_jedi_etkf_solver.py | 4 +++ .../run_jedi_local_ensemble_da_executable.py | 4 +++ src/swell/tasks/task_questions.py | 6 ++++ src/swell/utilities/question_defaults.py | 30 ++++++++++++++++++- .../utilities/render_jedi_interface_files.py | 2 ++ 9 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_solver.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_solver.py index 6e3c7257f..c7bd0e7ad 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_solver.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_solver.py @@ -18,14 +18,17 @@ def ensemble_solver(template_dict: Mapping) -> Mapping: local_ensemble_inflation_rtps = template_dict['local_ensemble_inflation_rtps'] local_ensemble_inflation_rtpp = template_dict['local_ensemble_inflation_rtpp'] local_ensemble_inflation_mult = template_dict['local_ensemble_inflation_mult'] + frac_retained_variance = template_dict['vertical_localization_frac_retained_variance'] + vertical_localization_lengthscale = template_dict['vertical_localization_lengthscale'] + vert_loc_unit = template_dict['vertical_localization_unit'] ensemble_solver = { 'solver': local_ensemble_solver, 'use linear observer': local_ensemble_use_linear_observer, 'vertical localization': { - 'fraction of retained variance': 0.5, - 'lengthscale': 1.5, - 'lengthscale units': 'logp' + 'fraction of retained variance': frac_retained_variance, + 'lengthscale': vertical_localization_lengthscale, + 'lengthscale units': vert_loc_unit }, 'inflation': { 'rtps': local_ensemble_inflation_rtps, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 9eb9d343c..d2f0a0749 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -353,8 +353,17 @@ vertical_localization_ioda_vertical_coord_group: options: - MetaData +vertical_localization_unit: + default_value: 'logp' + options: + - 'levels' + - 'logp' + vertical_localization_lengthscale: - default_value: 4 + default_value: 2 + +vertical_localization_frac_retained_variance: + default_value: 0.95 vertical_localization_method: default_value: Vertical localization diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index 9d89de48b..794be5a84 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -47,6 +47,9 @@ class SuiteConfig(QuestionContainer, Enum): qd.skip_ensemble_hofx(True), qd.local_ensemble_solver("Deterministic GETKF"), qd.local_ensemble_use_linear_observer(True), + qd.vertical_localization_unit('logp'), + qd.vertical_localization_lengthscale(2.0), + qd.vertical_localization_frac_retained_variance(0.95), qd.ensmean_only(False), qd.local_ensemble_save_posterior_mean(True), qd.local_ensemble_save_posterior_mean_increment(True), @@ -91,6 +94,9 @@ class SuiteConfig(QuestionContainer, Enum): qd.skip_ensemble_hofx(True), qd.local_ensemble_solver("Deterministic GETKF"), qd.local_ensemble_use_linear_observer(True), + qd.vertical_localization_unit('logp'), + qd.vertical_localization_lengthscale(2.0), + qd.vertical_localization_frac_retained_variance(0.95), qd.ensmean_only(False), qd.local_ensemble_save_posterior_mean(True), qd.local_ensemble_save_posterior_mean_increment(True), diff --git a/src/swell/tasks/run_jedi_etkf_observer.py b/src/swell/tasks/run_jedi_etkf_observer.py index 4b80ed3ec..b4cb1c530 100644 --- a/src/swell/tasks/run_jedi_etkf_observer.py +++ b/src/swell/tasks/run_jedi_etkf_observer.py @@ -87,8 +87,12 @@ def execute(self) -> None: self.config.vertical_localization_method()) self.jedi_rendering.add_key('vertical_localization_apply_log_transform', self.config.vertical_localization_apply_log_transform()) + self.jedi_rendering.add_key('vertical_localization_unit', + self.config.vertical_localization_unit()) self.jedi_rendering.add_key('vertical_localization_lengthscale', self.config.vertical_localization_lengthscale()) + self.jedi_rendering.add_key('vertical_localization_frac_retained_variance', + self.config.vertical_localization_frac_retained_variance()) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord', self.config.vertical_localization_ioda_vertical_coord()) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord_group', diff --git a/src/swell/tasks/run_jedi_etkf_solver.py b/src/swell/tasks/run_jedi_etkf_solver.py index 0e7bf6245..fe5972b57 100644 --- a/src/swell/tasks/run_jedi_etkf_solver.py +++ b/src/swell/tasks/run_jedi_etkf_solver.py @@ -86,8 +86,12 @@ def execute(self) -> None: self.config.vertical_localization_method()) self.jedi_rendering.add_key('vertical_localization_apply_log_transform', self.config.vertical_localization_apply_log_transform()) + self.jedi_rendering.add_key('vertical_localization_unit', + self.config.vertical_localization_unit()) self.jedi_rendering.add_key('vertical_localization_lengthscale', self.config.vertical_localization_lengthscale()) + self.jedi_rendering.add_key('vertical_localization_frac_retained_variance', + self.config.vertical_localization_frac_retained_variance()) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord', self.config.vertical_localization_ioda_vertical_coord()) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord_group', diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 7254e50a2..052caa442 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -93,8 +93,12 @@ def execute(self) -> None: self.config.vertical_localization_method()) self.jedi_rendering.add_key('vertical_localization_apply_log_transform', self.config.vertical_localization_apply_log_transform()) + self.jedi_rendering.add_key('vertical_localization_unit', + self.config.vertical_localization_unit()) self.jedi_rendering.add_key('vertical_localization_lengthscale', self.config.vertical_localization_lengthscale()) + self.jedi_rendering.add_key('vertical_localization_frac_retained_variance', + self.config.vertical_localization_frac_retained_variance()) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord', self.config.vertical_localization_ioda_vertical_coord()) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord_group', diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index b0617b130..8197b3905 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -766,6 +766,8 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_ioda_vertical_coord(), qd.vertical_localization_ioda_vertical_coord_group(), qd.vertical_localization_lengthscale(), + qd.vertical_localization_unit(), + qd.vertical_localization_frac_retained_variance(), qd.vertical_localization_method(), qd.perhost(), qd.change_vbc_to_sbc(), @@ -823,6 +825,8 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_ioda_vertical_coord(), qd.vertical_localization_ioda_vertical_coord_group(), qd.vertical_localization_lengthscale(), + qd.vertical_localization_unit(), + qd.vertical_localization_frac_retained_variance(), qd.vertical_localization_method(), qd.perhost(), qd.change_vbc_to_sbc(), @@ -860,6 +864,8 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_ioda_vertical_coord(), qd.vertical_localization_ioda_vertical_coord_group(), qd.vertical_localization_lengthscale(), + qd.vertical_localization_unit(), + qd.vertical_localization_frac_retained_variance(), qd.vertical_localization_method(), qd.perhost(), qd.change_vbc_to_sbc(), diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index a595be4e3..c63c13e29 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -1742,7 +1742,35 @@ class vertical_localization_lengthscale(TaskQuestion): "geos_atmosphere" ]) prompt: str = "What is the length scale for vertical covariance localization?" - widget_type: WType = WType.INTEGER + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_frac_retained_variance(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_frac_retained_variance" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the fraction of vertical retained variance for GETKF?" + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_unit (TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_unit" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the vertical localization unit for GETKF?" + widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index e303d754a..1306f2729 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -129,6 +129,8 @@ def __init__( 'vertical_localization_ioda_vertical_coord', 'vertical_localization_ioda_vertical_coord_group', 'vertical_localization_lengthscale', + 'vertical_localization_unit', + 'vertical_localization_frac_retained_variance', 'vertical_localization_method', 'vertical_resolution', 'window_begin', From 591a7ea3a4162326183acc0ba8ef1640339d6755 Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Fri, 7 Aug 2026 20:06:27 -0400 Subject: [PATCH 7/7] restore suite/localensembleda create suite/localensembleda_2steps --- src/swell/suites/localensembleda/flow.cylc | 32 +- .../suites/localensembleda/suite_config.py | 50 +- .../eva/increment-geos_atmosphere.yaml | 703 ++++++++++++++++++ .../eva/observations-geos_atmosphere.yaml | 380 ++++++++++ .../suites/localensembleda_2steps/flow.cylc | 259 +++++++ .../localensembleda_2steps/suite_config.py | 126 ++++ src/swell/utilities/yaml_utils.py | 1 - 7 files changed, 1507 insertions(+), 44 deletions(-) create mode 100644 src/swell/suites/localensembleda_2steps/eva/increment-geos_atmosphere.yaml create mode 100644 src/swell/suites/localensembleda_2steps/eva/observations-geos_atmosphere.yaml create mode 100644 src/swell/suites/localensembleda_2steps/flow.cylc create mode 100644 src/swell/suites/localensembleda_2steps/suite_config.py diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index 0bebb0f57..e62840326 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -64,10 +64,9 @@ GetEnsembleGeosExperiment-{{model_component}} => sync_point + sync_point => RunJediObsfiltersExecutable-{{model_component}} {% if skip_ensemble_hofx %} - sync_point => RunJediObsfiltersExecutable-{{model_component}} - RunJediObsfiltersExecutable-{{model_component}} => RunJediEtkfObserver-{{model_component}} - RunJediEtkfObserver-{{model_component}} => RunJediEtkfSolver-{{model_component}} + sync_point => RunJediObsfiltersExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} {% else %} # Run hofx for ensemble members according to strategy {% if ensemble_hofx_strategy == 'serial' %} @@ -90,17 +89,17 @@ # EvaIncrement - RunJediEtkfSolver-{{model_component}} => EvaIncrement-{{model_component}} + RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaIncrement-{{model_component}} # EvaObservations - RunJediEtkfSolver-{{model_component}} => EvaObservations-{{model_component}} + # RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} # Save observations - # RunJediEtkfSolver-{{model_component}} => SaveObsDiags-{{model_component}} + # RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} # Clean up large files # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & - # EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} + EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} @@ -215,24 +214,6 @@ --{{key}} = {{value}} {%- endfor %} - [[RunJediEtkfObserver-{{model_component}}]] - script = "swell task RunJediEtkfObserver $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediEtkfObserver"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediEtkfObserver"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunJediEtkfSolver-{{model_component}}]] - script = "swell task RunJediEtkfSolver $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediEtkfSolver"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediEtkfSolver"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - [[EvaIncrement-{{model_component}}]] script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" @@ -254,6 +235,7 @@ script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" {% endfor %} + [[sync_point]] script = true # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index 312b579dc..e80f67cb0 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -39,30 +39,24 @@ class SuiteConfig(QuestionContainer, Enum): 'rtodling/archive/Restarts/JEDI/541x'), qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' 'rtodling/archive/541/Milan'), - qd.npx_proc(4), - qd.npy_proc(4), + qd.npx_proc(3), + qd.npy_proc(3), qd.cycle_times(['T00']), qd.ensemble_num_members(3), qd.skip_ensemble_hofx(True), qd.local_ensemble_solver("Deterministic GETKF"), - qd.local_ensemble_use_linear_observer(True), - qd.vertical_localization_unit('logp'), - qd.vertical_localization_lengthscale(2.0), - qd.vertical_localization_frac_retained_variance(0.95), + qd.local_ensemble_use_linear_observer(False), qd.ensmean_only(False), qd.local_ensemble_save_posterior_mean(True), qd.local_ensemble_save_posterior_mean_increment(True), qd.local_ensemble_save_posterior_ensemble(False), qd.local_ensemble_save_posterior_ensemble_increments(False), - qd.obs_thinning_rej_fraction(0.9), + qd.obs_thinning_rej_fraction(0.75), qd.observations([ - "sondes", - "sfcship", "atms_n20", ]), qd.window_length("PT6H"), qd.window_type("3D"), - qd.change_vbc_to_sbc(False), qd.clean_patterns(['*.txt']) ] ) @@ -85,17 +79,14 @@ class SuiteConfig(QuestionContainer, Enum): 'rtodling/archive/Restarts/JEDI/541x'), qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' 'rtodling/archive/541/Milan'), - qd.npx_proc(8), - qd.npy_proc(8), - qd.perhost(96), + qd.npx_proc(4), + qd.npy_proc(4), + # qd.perhost(32), qd.cycle_times(['T00']), qd.ensemble_num_members(16), qd.skip_ensemble_hofx(True), qd.local_ensemble_solver("Deterministic GETKF"), qd.local_ensemble_use_linear_observer(True), - qd.vertical_localization_unit('logp'), - qd.vertical_localization_lengthscale(2.0), - qd.vertical_localization_frac_retained_variance(0.95), qd.ensmean_only(False), qd.local_ensemble_save_posterior_mean(True), qd.local_ensemble_save_posterior_mean_increment(True), @@ -103,13 +94,36 @@ class SuiteConfig(QuestionContainer, Enum): qd.local_ensemble_save_posterior_ensemble_increments(False), qd.obs_thinning_rej_fraction(0.75), qd.observations([ + "aircraft_temperature", + "aircraft_wind", "sondes", - "sfcship", + "gps", + "amsua_aqua", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "amsr2_gcom-w1", "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n18", + "avhrr3_n19", + "scatwind", + "sfcship", + "sfc", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "mls55_aura", + "omi_aura", + "ompsnm_npp", + "pibal", + "ssmis_f17", + "amsua_metop-b", + "amsua_metop-c" ]), qd.window_length("PT6H"), qd.window_type("3D"), - qd.change_vbc_to_sbc(False), qd.clean_patterns(['*.txt']) ] ) diff --git a/src/swell/suites/localensembleda_2steps/eva/increment-geos_atmosphere.yaml b/src/swell/suites/localensembleda_2steps/eva/increment-geos_atmosphere.yaml new file mode 100644 index 000000000..6e4a0fd86 --- /dev/null +++ b/src/swell/suites/localensembleda_2steps/eva/increment-geos_atmosphere.yaml @@ -0,0 +1,703 @@ +datasets: + +- group: increment + type: LatLon + filename: {{increment_file_path}} + name: experiment_increment + variables: [ua, va, t, q, lat, lon] + +graphics: + + plotting_backend: Emcpy + figure_list: + + #map plot for temperature increment (lowest level) + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[71,...]' + label: T increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[62,...]' + label: T increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[49,...]' + label: T increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[42,...]' + label: T increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[24,...]' + label: T increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[14,...]' + label: T increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment (lowest level) + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[71,...]' + label: U increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[62,...]' + label: U increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[49,...]' + label: U increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[42,...]' + label: U increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[24,...]' + label: U increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[14,...]' + label: U increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment (lowest level) + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[71,...]' + label: V increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[62,...]' + label: V increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[49,...]' + label: V increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[42,...]' + label: V increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[24,...]' + label: V increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[14,...]' + label: V increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Specific Humidity increment (lowest level) + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[71,...]' + label: Q increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[62,...]' + label: Q increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[49,...]' + label: Q increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[42,...]' + label: Q increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[24,...]' + label: Q increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 diff --git a/src/swell/suites/localensembleda_2steps/eva/observations-geos_atmosphere.yaml b/src/swell/suites/localensembleda_2steps/eva/observations-geos_atmosphere.yaml new file mode 100644 index 000000000..9a3d862da --- /dev/null +++ b/src/swell/suites/localensembleda_2steps/eva/observations-geos_atmosphere.yaml @@ -0,0 +1,380 @@ +datasets: + +- name: experiment + type: IodaObsSpace + filenames: + - {{obs_path_file}} + channels: &channels {{channels}} + groups: + - name: ObsValue + variables: &variables {{simulated_variables}} + - name: ombg + - name: oman + - name: EffectiveQC0 + - name: EffectiveQC1 + - name: GsiHofXBc + - name: GsiEffectiveQC + - name: MetaData + - name: hofx_y_mean_xb0 + - name: hofx_y_mean_xb1 + - name: hofx0_1 + - name: hofx1_1 + +transforms: + +# Generate Increment for JEDI +- transform: arithmetic + new name: experiment::increment::${variable} + equals: experiment::ombg::${variable}-experiment::oman::${variable} + for: + variable: *variables + +# Generate omb for GSI +- transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + +# Generate ensemble mean omb for JEDI +- transform: arithmetic + new name: experiment::ObsValueMinusMeanHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx_y_mean_xb0::${variable} + for: + variable: *variables + +# Generate ensemble mean hofx difference +- transform: arithmetic + new name: experiment::MeanHofxMinusGsiHofXBc::${variable} + equals: experiment::hofx_y_mean_xb0::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + +# Generate mean hofx that passed QC for JEDI +- transform: accept where + new name: experiment::meanHofxPassedQc::${variable} + starting field: experiment::hofx_y_mean_xb0::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate GSI hofx that passed JEDI QC0 +- transform: accept where + new name: experiment::GsiHofXBcPassedQc::${variable} + starting field: experiment::GsiHofXBc::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate ensemble mean omb that passed QC for JEDI +- transform: accept where + new name: experiment::ObsValueMinushofxPassedQc::${variable} + starting field: experiment::ObsValueMinusMeanHofx::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate omb that passed QC0 for GSI +- transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +graphics: + + plotting_backend: Emcpy + figure_list: + + # Correlation scatter plots + # ------------------------- + + # JEDI h(x) vs Observations + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'Observations vs. JEDI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/jedi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx_y_mean_xb0::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::meanHofxPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus obs (passed QC in JEDI)' + + # GSI h(x) vs Observations + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'Observations vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'GSI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBc::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBcPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'GSI h(x) versus obs (passed QC in JEDI)' + + # JEDI h(x) vs GSI h(x) + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_jedi_hofx_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::GsiHofXBc::${variable} + y: + variable: experiment::hofx_y_mean_xb0::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus GSI h(x)' + - type: Scatter + x: + variable: experiment::GsiHofXBcPassedQc::${variable} + y: + variable: experiment::meanHofxPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' + + # JEDI omb vs GSI omb + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI omb vs. GSI omb | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_omb_vs_jedi_omb_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'GSI observation minus h(x)' + add_ylabel: 'JEDI observation minus h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + y: + variable: experiment::ObsValueMinusMeanHofx::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI omb vs JEDI omb (all obs)' + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + y: + variable: experiment::ObsValueMinushofxPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'GSI omb vs JEDI omb (passed QC in JEDI)' +# Map plots# --------- + + # Observations + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValue::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'Observations | {{instrument_title}} | Obs Value' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/observations_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: ObsValue + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValue::${variable} + channel: ${channel} + markersize: 2 + label: ObsValue + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # omb jedi + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValueMinusMeanHofx::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'JEDI OmB | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_jedi_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValueMinusMeanHofx::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # omb gsi + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValueMinusGsiHofXBc::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'GSI OmB | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_gsi_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # mean hofx difference + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::MeanHofxMinusGsiHofXBc::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'Hofx Difference | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/hofx_difference_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::MeanHofxMinusGsiHofXBc::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} diff --git a/src/swell/suites/localensembleda_2steps/flow.cylc b/src/swell/suites/localensembleda_2steps/flow.cylc new file mode 100644 index 000000000..0bebb0f57 --- /dev/null +++ b/src/swell/suites/localensembleda_2steps/flow.cylc @@ -0,0 +1,259 @@ +#!jinja2 +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based LocalEnsembleDA Algorithm + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +{{scheduling['stall_timeout']}} + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} + # ------------------ + + # Perform staging that is cycle dependent + BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point + + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + + RenderJediObservations-{{model_component}} => sync_point + + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => RenderJediObservations-{{model_component}} + + GetEnsembleGeosExperiment-{{model_component}} => sync_point + + {% if skip_ensemble_hofx %} + sync_point => RunJediObsfiltersExecutable-{{model_component}} + RunJediObsfiltersExecutable-{{model_component}} => RunJediEtkfObserver-{{model_component}} + RunJediEtkfObserver-{{model_component}} => RunJediEtkfSolver-{{model_component}} + {% else %} + # Run hofx for ensemble members according to strategy + {% if ensemble_hofx_strategy == 'serial' %} + sync_point => RunJediEnsembleMeanVariance-{{model_component}} => RunJediHofxEnsembleExecutable-{{model_component}} + RunJediHofxEnsembleExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + + {% elif ensemble_hofx_strategy == 'parallel' %} + {% for packet in range(ensemble_hofx_packets) %} + # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet + + # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder + # RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunEnsembleHofxCombiner-{{model_component}} + # RunEnsembleHofxCombiner-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + + sync_point => RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} + RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + {% endfor %} + {% endif %} + {% endif %} + + + # EvaIncrement + RunJediEtkfSolver-{{model_component}} => EvaIncrement-{{model_component}} + + # EvaObservations + RunJediEtkfSolver-{{model_component}} => EvaObservations-{{model_component}} + + # Save observations + # RunJediEtkfSolver-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & + # EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% for model_component in model_components %} + + [[CloneGeosMksi-{{model_component}}]] + script = "swell task CloneGeosMksi $config -m {{model_component}}" + + [[GenerateObservingSystemRecords-{{model_component}}]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[ GetBackground-{{model_component}} ]] + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetEnsembleGeosExperiment-{{model_component}}]] + script = "swell task GetEnsembleGeosExperiment $config -d $datetime -m {{model_component}}" + + [[RenderJediObservations-geos_atmosphere]] + script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" + + [[RunJediObsfiltersExecutable-{{model_component}}]] + script = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediObsfiltersExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediObsfiltersExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediEnsembleMeanVariance-{{model_component}}]] + script = "swell task RunJediEnsembleMeanVariance $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEnsembleMeanVariance"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEnsembleMeanVariance"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + + {% if not skip_ensemble_hofx %} + {% if ensemble_hofx_strategy == 'serial' %} + [[RunJediHofxEnsembleExecutable-{{model_component}}]] + script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + {% elif ensemble_hofx_strategy == 'parallel' %} + {% for packet in range(ensemble_hofx_packets) %} + [[RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}}]] + script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}} -p {{packet}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% endfor %} + {% endif %} + {% endif %} + + [[RunJediLocalEnsembleDaExecutable-{{model_component}}]] + script = "swell task RunJediLocalEnsembleDaExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediLocalEnsembleDaExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediLocalEnsembleDaExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediEtkfObserver-{{model_component}}]] + script = "swell task RunJediEtkfObserver $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEtkfObserver"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEtkfObserver"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediEtkfSolver-{{model_component}}]] + script = "swell task RunJediEtkfSolver $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEtkfSolver"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEtkfSolver"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = true +# EnKF not ready to use Eva +# script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" +# platform = {{platform}} +# execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} +# [[[directives]]] +# {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} +# --{{key}} = {{value}} +# {%- endfor %} + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + + [[sync_point]] + script = true +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda_2steps/suite_config.py b/src/swell/suites/localensembleda_2steps/suite_config.py new file mode 100644 index 000000000..312b579dc --- /dev/null +++ b/src/swell/suites/localensembleda_2steps/suite_config.py @@ -0,0 +1,126 @@ +# -------------------------------------------------------------------------------------------------- +# @package configuration +# +# Class containing the configuration. This is a dictionary that is converted from +# an input yaml configuration file. Various function are included for interacting with the +# dictionary. +# +# -------------------------------------------------------------------------------------------------- + + +from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.suites.suite_questions import SuiteQuestions as sq + +from enum import Enum + + +# -------------------------------------------------------------------------------------------------- + +class SuiteConfig(QuestionContainer, Enum): + + # -------------------------------------------------------------------------------------------------- + + localensembleda_tier1 = QuestionList( + list_name="localensembleda", + questions=[ + sq.marine, + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.skip_ensemble_hofx(), + qd.final_cycle_point("2023-10-10T12:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.horizontal_resolution('91'), + qd.background_experiment('x0050'), + qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/Restarts/JEDI/541x'), + qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/541/Milan'), + qd.npx_proc(4), + qd.npy_proc(4), + qd.cycle_times(['T00']), + qd.ensemble_num_members(3), + qd.skip_ensemble_hofx(True), + qd.local_ensemble_solver("Deterministic GETKF"), + qd.local_ensemble_use_linear_observer(True), + qd.vertical_localization_unit('logp'), + qd.vertical_localization_lengthscale(2.0), + qd.vertical_localization_frac_retained_variance(0.95), + qd.ensmean_only(False), + qd.local_ensemble_save_posterior_mean(True), + qd.local_ensemble_save_posterior_mean_increment(True), + qd.local_ensemble_save_posterior_ensemble(False), + qd.local_ensemble_save_posterior_ensemble_increments(False), + qd.obs_thinning_rej_fraction(0.9), + qd.observations([ + "sondes", + "sfcship", + "atms_n20", + ]), + qd.window_length("PT6H"), + qd.window_type("3D"), + qd.change_vbc_to_sbc(False), + qd.clean_patterns(['*.txt']) + ] + ) + + localensembleda_tier2 = QuestionList( + list_name="localensembleda", + questions=[ + sq.marine, + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.skip_ensemble_hofx(), + qd.final_cycle_point("2023-10-10T12:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.horizontal_resolution('91'), + qd.background_experiment('x0050'), + qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/Restarts/JEDI/541x'), + qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/541/Milan'), + qd.npx_proc(8), + qd.npy_proc(8), + qd.perhost(96), + qd.cycle_times(['T00']), + qd.ensemble_num_members(16), + qd.skip_ensemble_hofx(True), + qd.local_ensemble_solver("Deterministic GETKF"), + qd.local_ensemble_use_linear_observer(True), + qd.vertical_localization_unit('logp'), + qd.vertical_localization_lengthscale(2.0), + qd.vertical_localization_frac_retained_variance(0.95), + qd.ensmean_only(False), + qd.local_ensemble_save_posterior_mean(True), + qd.local_ensemble_save_posterior_mean_increment(True), + qd.local_ensemble_save_posterior_ensemble(False), + qd.local_ensemble_save_posterior_ensemble_increments(False), + qd.obs_thinning_rej_fraction(0.75), + qd.observations([ + "sondes", + "sfcship", + "atms_n20", + ]), + qd.window_length("PT6H"), + qd.window_type("3D"), + qd.change_vbc_to_sbc(False), + qd.clean_patterns(['*.txt']) + ] + ) + + # -------------------------------------------------------------------------------------------------- + + localensembleda = QuestionList( + list_name="localensembleda", + questions=[ + localensembleda_tier2 + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/yaml_utils.py b/src/swell/utilities/yaml_utils.py index 80aca65a3..c15810552 100644 --- a/src/swell/utilities/yaml_utils.py +++ b/src/swell/utilities/yaml_utils.py @@ -40,5 +40,4 @@ def replace_string_value(data, sa, sb): # Return integers, booleans, etc., as-is return data - # --------------------------------------------------------------------------------------------------