From 830db9613f51215125b25c076cb4f46c0a34564e Mon Sep 17 00:00:00 2001 From: jbopp Date: Mon, 28 Mar 2022 17:33:22 +0200 Subject: [PATCH 1/9] Handling exception if JCMgeo fails. Allowing to resume simulation series which do not have a HDF5 store by checking whether fieldbags already exist. --- pypmj/core.py | 84 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/pypmj/core.py b/pypmj/core.py index 251df16..d3c7570 100644 --- a/pypmj/core.py +++ b/pypmj/core.py @@ -809,7 +809,7 @@ def compute_geometry(self, **jcm_kwargs): """Computes the geometry (i.e. runs jcm.geo) for this simulation. The jcm_kwargs are directly passed to jcm.geo, except for `project_dir`, `keys` and `working_dir`, which are set automatically - (ignored if provided). + (ignored if provided). Returns False if JCMgeo fails, True otherwise. """ self.logger.debug('Computing geometry.') # Copy project to its working directory @@ -838,13 +838,20 @@ def compute_geometry(self, **jcm_kwargs): _thisdir = os.getcwd() os.chdir(self.project.working_dir) with utils.Capturing() as output: - jcm.geo(project_dir=self.project.working_dir, - keys=self.keys, - working_dir=self.project.working_dir, - **jcm_kwargs) + try: + jcm.geo(project_dir=self.project.working_dir, + keys=self.keys, + working_dir=self.project.working_dir, + **jcm_kwargs) + except RuntimeError as e: + self.logger.warn('Failed to compute geometry for simulation {}. JCMgeo returned "{}".'.format(self.number, str(e))) + return False + for line in output: logger_JCMgeo.debug(line) os.chdir(_thisdir) + + return True def solve_standalone(self, processing_func=None, wdir_mode='keep', run_post_process_files=None, resource_manager=None, @@ -1213,6 +1220,16 @@ class SimulationSet(object): the results and logs are kept for each simulation. Set this parameter to true to minimize the memory usage. Caution: you will loose all the `jcm_results` and `logs` in the `Simulation`-instances. + skip_existent_simulations_by_folder : bool, default False + Determines whether to skip simulations by the existence of a + project_results/fieldbag.jcm file in the storage folder of a respective + simulation with index i. The storage folder of a single simulation i is + given by the parameter `storage_folder` and a subfolder 'simulation[i]'. + Setting this parameter to True is handy if a simulation series is to be + continued when no HDF5 store is present. Simulation keys `geometry` and + `parameters` must not have changed when restarting the simulation series! + Otherwise, simulation indices might not match the assumed folder + structure any longer. """ # Names of the groups in the HDF5 store which are used to store metadata @@ -1224,13 +1241,14 @@ def __init__(self, project, keys, duplicate_path_levels=0, use_resultbag=False, transitional_storage_base=None, combination_mode='product', check_version_match=True, resource_manager=None, store_logs=False, - minimize_memory_usage=False): + minimize_memory_usage=False, skip_existent_simulations_by_folder=False): self.logger = logging.getLogger('core.' + self.__class__.__name__) # Save initialization arguments into namespace self.combination_mode = combination_mode self.store_logs = store_logs self.minimize_memory_usage = minimize_memory_usage + self.skip_existent_simulations_by_folder = skip_existent_simulations_by_folder # Analyze the provided keys self._check_keys(keys) @@ -1804,15 +1822,28 @@ def make_simulation_schedule(self, fix_h5_duplicated_rows=False): precheck = self._precheck_store() self.logger.debug('Result of the store pre-check: {}'.format(precheck)) - if precheck == 'Empty': - self.finished_sim_numbers = [] - if precheck == 'Extended Check': + if precheck == 'Empty' or self.skip_existent_simulations_by_folder: + stored_sim_numbers = [] + + if self.skip_existent_simulations_by_folder: + for i in range(self.num_sims): + simdir = _default_sim_wdir(self.storage_dir, self.simulations[i].number) + if os.path.exists(os.path.join(simdir, 'project_results/fieldbag.jcm')): + stored_sim_numbers.append(i) + + self.finished_sim_numbers = stored_sim_numbers + if len(self.finished_sim_numbers) > 0: + self.logger.info('Ignoring HDF5 store. Determining already finished simulations by ' + + 'existence of fieldbag.jcm instead. Number of found simulations: {}'.format( + len(self.finished_sim_numbers))) + + if precheck == 'Extended Check' and not self.skip_existent_simulations_by_folder: self.logger.info('Running extended check ...') self._extended_store_check() self.logger.info('Found matches in the extended check of the ' + 'HDF5 store. Number of stored simulations: {}'. format(len(self.finished_sim_numbers))) - elif precheck == 'Match': + elif precheck == 'Match' and not self.skip_existent_simulations_by_folder: stored_sim_numbers = list(self.get_store_data().index) if len(stored_sim_numbers) > self.num_sims: if fix_h5_duplicated_rows: @@ -2290,7 +2321,7 @@ def _resources_ready(self): def compute_geometry(self, simulation, **jcm_kwargs): """Computes the geometry (i.e. runs jcm.geo) for a specific simulation - of the simulation set. + of the simulation set. Returns False in case of an error, True otherwise. Parameters ---------- simulation : Simulation or int @@ -2307,10 +2338,10 @@ def compute_geometry(self, simulation, **jcm_kwargs): raise ValueError('`simulation` must be a Simulation of the ' + 'current SimulationSet or a simulation index' + ' (int).') - return + return False # Call the compute_geometry-method of the simulation - simulation.compute_geometry(**jcm_kwargs) + return simulation.compute_geometry(**jcm_kwargs) def solve_single_simulation(self, simulation, compute_geometry=True, run_post_process_files=None, @@ -2462,17 +2493,26 @@ def _start_simulations(self, N='all', processing_func=None, # Start the simulation if it is not already finished if not sim.number in self.finished_sim_numbers: # Compute the geometry if necessary + geo_succeeded = True if sim.rerun_JCMgeo or force_geo_run: - self.compute_geometry(sim, **jcm_geo_kwargs) - force_geo_run = False + if self.compute_geometry(sim, **jcm_geo_kwargs): + force_geo_run = False + else: + geo_succeeded = False; - # Start to solve the simulation and receive a job ID - job_id = sim.solve(**jcm_solve_kwargs) - self.logger.debug( - 'Queued simulation {0} of {1} with job_id {2}'. - format(i + 1, self.num_sims, sim.job_id)) - job_ids.append(job_id) - ids_to_sim_number[job_id] = sim.number + if geo_succeeded: + # Start to solve the simulation and receive a job ID + job_id = sim.solve(**jcm_solve_kwargs) + self.logger.debug( + 'Queued simulation {0} of {1} with job_id {2}'. + format(i + 1, self.num_sims, sim.job_id)) + job_ids.append(job_id) + ids_to_sim_number[job_id] = sim.number + else: + sim.status = 'Skipped' + self.logger.debug( + 'Skipping simulation {0} of {1}'. + format(i + 1, self.num_sims)) else: # Set `force_geo_run` to True if this finished simulation would # have caused to compute the geometry From 485df6c28fe7435b2466d3051a183dca9f576eba Mon Sep 17 00:00:00 2001 From: jbopp Date: Tue, 4 Oct 2022 17:57:17 +0200 Subject: [PATCH 2/9] Skip simulations where meshing failed during optimization --- pypmj/optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypmj/optimizer.py b/pypmj/optimizer.py index 59a7e52..bb0f476 100644 --- a/pypmj/optimizer.py +++ b/pypmj/optimizer.py @@ -116,7 +116,7 @@ def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date sid = simuset.simulation_properties['suggestion_id'][i] # The simulations has failed. Skip it. - if simuset.simulations[i].exit_code != 0: + if not hasattr(simuset.simulations[i], "exit_code") or simuset.simulations[i].exit_code != 0: self.study.clear_suggestion(sid, 'Simulation failed.') self.logger.warn('Simulation with suggestion_id {} failed. Ignoring and continuing...'.format(sid)) continue From 7ff7a40c91135be9874b646324c5c52f6775c921 Mon Sep 17 00:00:00 2001 From: jbopp Date: Tue, 25 Oct 2022 17:49:16 +0200 Subject: [PATCH 3/9] Optimizer can now handle parameter sweeps and apply objective function to set of simulations --- pypmj/optimizer.py | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/pypmj/optimizer.py b/pypmj/optimizer.py index bb0f476..0d16037 100644 --- a/pypmj/optimizer.py +++ b/pypmj/optimizer.py @@ -19,6 +19,10 @@ class Optimizer(object): List of constraints to be applied to the parameters to be optimized. Refer to JCMsuite's Python command reference for function jcmwave.optimizer.create_study(). constant_keys : dict, default {} Dict of template keys that are constant for all simulations. This dict will be assigned to the `constants` key of the `keys` parameter passed to the constructor of class pypmj.core.SimulationSet. + parameter_keys : dict, default {} + Dict of template keys that are not constant for all simulations and that do not affect the geometry. This dict will be assigned to the `parameters` key of the `keys` parameter passed to the constructor of class pypmj.core.SimulationSet. If this dict is not empty, a respective parameter sweep will be performed in list-mode for each optimizer suggestion. The objective function will be called once for each sweep. + geometry_keys : dict, default {} + Dict of template keys that are not constant for all simulations and that affect the geometry. This dict will be assigned to the `geometry` key of the `keys` parameter passed to the constructor of class pypmj.core.SimulationSet. If this dict is not empty, a respective parameter sweep will be performed in list-mode for each optimizer suggestion. The objective function will be called once for each sweep. max_iter : int, default 20 Maximum number of simulations to run in total. Refer to JCMsuite's Python command reference for function jcmwave.client.Study.set_parameters(). num_parallel : int, default 0 @@ -26,13 +30,15 @@ class Optimizer(object): jcm_create_study_kwargs : dict, default {} Additional arguments passed to jcmwave.optimizer.create_study(). """ - def __init__(self, project, domain, constraints=[], constant_keys={}, max_iter=20, num_parallel=0, jcm_create_study_kwargs={}): + def __init__(self, project, domain, constraints=[], constant_keys={}, parameter_keys={}, geometry_keys={}, max_iter=20, num_parallel=0, jcm_create_study_kwargs={}): # Initialize members self.logger = logging.getLogger('core.' + self.__class__.__name__) self.__project = project self.domain = domain self.constraints = constraints self.constant_keys = constant_keys + self.parameter_keys = parameter_keys + self.geometry_keys = geometry_keys self.max_iter = max_iter self.__num_parallel = num_parallel if num_parallel > 0 else ResourceManager().get_current_resources().get_resources()[0].multiplicity @@ -77,12 +83,14 @@ def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date pass_ccosts_to_processing_func : bool, default False Refer to function pypmj.core.SimulationSet.run(). """ + is_sweep = len(self.parameter_keys) > 0 or len(self.geometry_keys) > 0 + # Continue if study has not finished yet. while (not self.study.is_done()): # Obtain suggestions for the amount of simulations which should run in parallel. suggestions = [] suggestion_ids = [] - for i in range(self.__num_parallel): + for i in (range(self.__num_parallel) if not is_sweep else range(1)): suggestions.append(self.study.get_suggestion()) suggestion_ids.append(suggestions[i].id) if self.study.info()['is_done']: @@ -90,13 +98,14 @@ def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date # Build template keys from suggestions and from given constant keys. # Suggestion IDs are passed as a parameter key for later identification. - parameter_keys = {'suggestion_id': suggestion_ids} - geometry_keys = dict() + parameter_keys = self.parameter_keys + parameter_keys['suggestion_id'] = suggestion_ids if not is_sweep else suggestion_ids[0] + geometry_keys = self.geometry_keys for key in self.__domain_keys: values = [] for suggestion in suggestions: values.append(suggestion.kwargs[key]) - geometry_keys[key] = values + geometry_keys[key] = values if not is_sweep else values[0] template_keys = { 'constants': self.constant_keys, @@ -112,22 +121,28 @@ def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date self.__clear_storage_dir(simuset) # Simulations have finished. Loop through the results. - for i in range(len(simuset.simulations)): + for i in (range(len(simuset.simulations)) if not is_sweep else range(1)): sid = simuset.simulation_properties['suggestion_id'][i] - # The simulations has failed. Skip it. - if not hasattr(simuset.simulations[i], "exit_code") or simuset.simulations[i].exit_code != 0: - self.study.clear_suggestion(sid, 'Simulation failed.') - self.logger.warn('Simulation with suggestion_id {} failed. Ignoring and continuing...'.format(sid)) - continue + # The simulation(s) has/have failed. Skip this suggestion. + if not is_sweep: + if not hasattr(simuset.simulations[i], "exit_code") or simuset.simulations[i].exit_code != 0: + self.study.clear_suggestion(sid, 'Simulation failed.') + self.logger.warn('Simulation with suggestion_id {} failed. Ignoring and continuing...'.format(sid)) + continue + else: + if all([not hasattr(x, "exit_code") or x.exit_code != 0 for x in simuset.simulations]): + self.study.clear_suggestion(sid, 'Simulation set failed.') + self.logger.warn('All simulations within set with suggestion_id {} failed. Ignoring and continuing...'.format(sid)) + continue - # Calculate the objective value for the simulation. Skip it if None is returned. - observed_result = objective_func(simuset.simulations[i]); + # Calculate the objective value of the simulation (set). Skip it if None is returned. + observed_result = objective_func([simuset.simulations[i]] if not is_sweep else simuset.simulations); if observed_result is None: self.study.clear_suggestion(sid, 'Simulation skipped by client.') continue - # Pass objective value for the simulation to the study object. + # Pass objective value of the simulation (set) to the study object. observation = self.study.new_observation() observation.add(observed_result) self.study.add_observation(observation, sid) From f3023ac6e9b7a612f0cb750507665a08efcf858e Mon Sep 17 00:00:00 2001 From: jbopp Date: Tue, 17 Jan 2023 20:10:19 +0100 Subject: [PATCH 4/9] Take into account that sometimes the exit_code is 0 even if the simulations failed. --- pypmj/optimizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypmj/optimizer.py b/pypmj/optimizer.py index 0d16037..4872616 100644 --- a/pypmj/optimizer.py +++ b/pypmj/optimizer.py @@ -126,12 +126,12 @@ def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date # The simulation(s) has/have failed. Skip this suggestion. if not is_sweep: - if not hasattr(simuset.simulations[i], "exit_code") or simuset.simulations[i].exit_code != 0: + if not hasattr(simuset.simulations[i], "status") or simuset.simulations[i].status == 'Failed': self.study.clear_suggestion(sid, 'Simulation failed.') self.logger.warn('Simulation with suggestion_id {} failed. Ignoring and continuing...'.format(sid)) continue else: - if all([not hasattr(x, "exit_code") or x.exit_code != 0 for x in simuset.simulations]): + if all([not hasattr(x, "status") or x.status == 'Failed' for x in simuset.simulations]): self.study.clear_suggestion(sid, 'Simulation set failed.') self.logger.warn('All simulations within set with suggestion_id {} failed. Ignoring and continuing...'.format(sid)) continue From c9c5a51be7dad7f3b669a45cceed7b30e246739f Mon Sep 17 00:00:00 2001 From: jbopp Date: Fri, 3 Mar 2023 12:14:30 +0100 Subject: [PATCH 5/9] Allow to keep simulation results in Optimizer module --- pypmj/optimizer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pypmj/optimizer.py b/pypmj/optimizer.py index 4872616..155f83e 100644 --- a/pypmj/optimizer.py +++ b/pypmj/optimizer.py @@ -50,7 +50,7 @@ def __init__(self, project, domain, constraints=[], constant_keys={}, parameter_ self.study = jcm.optimizer.create_study(domain=self.domain, constraints=self.constraints, **jcm_create_study_kwargs) self.study.set_parameters(max_iter=self.max_iter, num_parallel=self.__num_parallel) - def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date', storage_base='from_config', use_resultbag=False, transitional_storage_base=None, resource_manager=None, minimize_memory_usage=False, processing_func=None, auto_rerun_failed=1, run_post_process_files=None, additional_keys=None, jcm_solve_kwargs=None, pass_ccosts_to_processing_func=False): + def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date', storage_base='from_config', use_resultbag=False, transitional_storage_base=None, resource_manager=None, minimize_memory_usage=False, processing_func=None, auto_rerun_failed=1, run_post_process_files=None, additional_keys=None, wdir_mode='delete', jcm_solve_kwargs=None, pass_ccosts_to_processing_func=False): """Runs the entire optimization study. Parameters ---------- @@ -78,6 +78,8 @@ def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date Refer to function pypmj.core.SimulationSet.run(). additional_keys : dict, default None Refer to function pypmj.core.SimulationSet.run(). + wdir_mode : {'keep', 'zip', 'delete'}, default 'delete' + Refer to function pypmj.core.SimulationSet.run(). If 'zip', the working directories are stored as 'simulations_[suggestion_id].zip' in `storage_folder`. jcm_solve_kwargs : dict, default None Refer to function pypmj.core.SimulationSet.run(). pass_ccosts_to_processing_func : bool, default False @@ -116,7 +118,8 @@ def run(self, objective_func, duplicate_path_levels=0, storage_folder='from_date # Initialize a SimulationSet and run the simulations. simuset = SimulationSet(self.__project, template_keys, combination_mode='list', duplicate_path_levels=duplicate_path_levels, storage_folder=storage_folder, storage_base=storage_base, use_resultbag=use_resultbag, transitional_storage_base=transitional_storage_base, resource_manager=resource_manager, minimize_memory_usage=minimize_memory_usage) simuset.make_simulation_schedule() - simuset.run(processing_func=processing_func, auto_rerun_failed=auto_rerun_failed, run_post_process_files=run_post_process_files, additional_keys=additional_keys, wdir_mode='delete', jcm_solve_kwargs=jcm_solve_kwargs, pass_ccosts_to_processing_func=pass_ccosts_to_processing_func) + zip_file_path = os.path.join(simuset.storage_dir, "simulations_{}.zip".format(suggestion_ids[0])) + simuset.run(processing_func=processing_func, auto_rerun_failed=auto_rerun_failed, run_post_process_files=run_post_process_files, additional_keys=additional_keys, wdir_mode=wdir_mode, zip_file_path=zip_file_path, jcm_solve_kwargs=jcm_solve_kwargs, pass_ccosts_to_processing_func=pass_ccosts_to_processing_func) simuset.close_store() self.__clear_storage_dir(simuset) From ead98230217ecc7ac5a6de23ff417af94a47eafa Mon Sep 17 00:00:00 2001 From: jbopp Date: Tue, 7 Mar 2023 11:37:33 +0100 Subject: [PATCH 6/9] Parameter combinations constituting a simulation set can be constraint by an arbitrary function. --- pypmj/core.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pypmj/core.py b/pypmj/core.py index d3c7570..0056a13 100644 --- a/pypmj/core.py +++ b/pypmj/core.py @@ -1794,7 +1794,7 @@ def fix_h5_store(self, try_restructure=True, brute_force=False): self.open_store() self.logger.info('Successfully restructured HDF5 store.') - def make_simulation_schedule(self, fix_h5_duplicated_rows=False): + def make_simulation_schedule(self, fix_h5_duplicated_rows=False, constraint_func=None): """Makes a schedule by getting a list of simulations that must be performed, reorders them to avoid unnecessary calls of JCMgeo, and checks the HDF5 store for simulation data which is already known. @@ -1802,9 +1802,12 @@ def make_simulation_schedule(self, fix_h5_duplicated_rows=False): case, you can rerun `make_simulation_schedule` with `fix_h5_duplicated_rows=True` to try to automatically fix it. Alternatively, you could call the `fix_h5_store`-method yourself. - + `constraint_func` is a function taking a set of parameter and + geometry keys as a dictionary. It should return True if a + simulation is to be performed for the respective set. False, if + the set is to be skipped. """ - self._get_simulation_list() + self._get_simulation_list(constraint_func) self._sort_simulations() # Init the failed simulation list @@ -1861,7 +1864,7 @@ def make_simulation_schedule(self, fix_h5_duplicated_rows=False): 'store. Number of stored simulations: {}'.format( len(self.finished_sim_numbers))) - def _get_simulation_list(self): + def _get_simulation_list(self, constraint_func): """Check the `parameters`- and `geometry`-dictionaries for sequences and generate a list which has a keys-dictionary for each distinct simulation by using the `combination_mode` as specified. @@ -1930,6 +1933,14 @@ def _get_simulation_list(self): propertyCombinations = [] for iSim in range(Nsims): propertyCombinations.append(tuple([l[iSim] for l in loopList])) + + # Remove combinations based on constraint_func. + if constraint_func is not None: + propertyCombinationsFiltered = [x for x in propertyCombinations if \ + constraint_func(dict((k, v) for k, v in x))] + self.logger.info("Removed {} combinations based on the given constraint function.".format( \ + len(propertyCombinations) - len(propertyCombinationsFiltered))) + propertyCombinations = propertyCombinationsFiltered self.num_sims = len(propertyCombinations) # total num of simulations if self.num_sims == 1: From a343ca8df99306e4df383bc8614a30de876dc0cc Mon Sep 17 00:00:00 2001 From: jbopp Date: Mon, 15 Dec 2025 17:47:40 +0100 Subject: [PATCH 7/9] Ensure compatibility with Python 3.11 --- pypmj/core.py | 10 +++++----- pypmj/utils.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pypmj/core.py b/pypmj/core.py index 0056a13..8a04630 100644 --- a/pypmj/core.py +++ b/pypmj/core.py @@ -48,12 +48,12 @@ # Set warning filters warnings.filterwarnings(action='ignore', - message= '.*The\\ Leaf.*is\\ exceeding\\ the\\' + \ - ' maximum\\ recommended\\ rowsize.*\\Z(?ms)') + message= '(?ms).*The\\ Leaf.*is\\ exceeding\\ the\\' + \ + ' maximum\\ recommended\\ rowsize.*\\Z') warnings.filterwarnings(action='ignore', - message= '.*your\\ performance\\ may\\ suffer\\ ' + \ + message= '(?ms).*your\\ performance\\ may\\ suffer\\ ' + \ 'as\\ PyTables\\ will\\ pickle\\ object\\ types\\' + \ - ' that\\ it\\ cannot.*\\Z(?ms)') + ' that\\ it\\ cannot.*\\Z') # Set text template for strings replacing class attributes deleted for # memory efficiency (occurs if `minimize_memory_usage=True` @@ -2106,7 +2106,7 @@ def __restore_from_meta_dframe(self, which): if len(vals) == 1: dict_[col] = vals.iat[0] else: - dict_[col] = pd.to_numeric(vals, errors='ignore').values + dict_[col] = pd.to_numeric(vals).values return dict_ def _store_metadata(self): diff --git a/pypmj/utils.py b/pypmj/utils.py index 666f74b..a6cd37b 100644 --- a/pypmj/utils.py +++ b/pypmj/utils.py @@ -230,7 +230,7 @@ def obj_to_fixed_length_Series(obj, length): return obj = np.array(obj, dtype=dtype) series = pd.Series(index=list(range(length)), dtype=dtype) - series[:] = np.NaN # Fixes bool-case + series[:] = np.nan # Fixes bool-case series.loc[:len(obj) - 1] = obj return series From 1dc3fd3ccece5d4cd55ba5229fccad68ee50a21c Mon Sep 17 00:00:00 2001 From: jbopp Date: Mon, 15 Dec 2025 18:20:14 +0100 Subject: [PATCH 8/9] Ensure compatibility with Python 3.11 --- pypmj/core.py | 2 +- pypmj/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pypmj/core.py b/pypmj/core.py index 8a04630..f9098b0 100644 --- a/pypmj/core.py +++ b/pypmj/core.py @@ -715,7 +715,7 @@ def process_results(self, processing_func=None, overwrite=False): # We try to call the processing_func now. If it fails or its results # are not of type dict, it is ignored and the user will be warned - signature = inspect.getargspec(processing_func) + signature = inspect.getfullargspec(processing_func) if len(signature.args) == 1: procargs = [jcm_results_to_pass] elif len(signature.args) == 2: diff --git a/pypmj/utils.py b/pypmj/utils.py index a6cd37b..61a1485 100644 --- a/pypmj/utils.py +++ b/pypmj/utils.py @@ -45,7 +45,7 @@ def is_callable(obj): """ if _IS_PYTHON3: - return isinstance(obj, collections.Callable) + return isinstance(obj, collections.abc.Callable) return callable(obj) def file_content(file_path): @@ -145,7 +145,7 @@ def assign_kwargs_to_functions(functions, kwargs, ignore_unmatched=True): # Try to inspect the input arguments of the functions try: - input_args = [inspect.getargspec(func)[0] for func in functions] + input_args = [inspect.getfullargspec(func)[0] for func in functions] except Exception as e: raise RuntimeError('Unable to inspect the input arguments of' + ' the provided functions. Esception: {}'.format(e)) From c4921f74417c8a8dedae2f5fb976a0c974085dec Mon Sep 17 00:00:00 2001 From: jbopp Date: Tue, 16 Dec 2025 16:56:05 +0100 Subject: [PATCH 9/9] Bugfix in computational_costs_to_flat_dict() regarding refinement loops of uneqal lengths and appending pandas data frames --- pypmj/utils.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/pypmj/utils.py b/pypmj/utils.py index 61a1485..a72b4ae 100644 --- a/pypmj/utils.py +++ b/pypmj/utils.py @@ -241,11 +241,12 @@ def computational_costs_to_flat_dict(ccosts, _sub=False): This is useful to store the computational costs in a pandas DataFrame. Keys which have sequence values with a length other than - 1 are converted to single values, while appending an underscore plus - index to the key. - + 1 are converted to single values, while only keeping the last value within + the sequence. Keeping all sequence values and appending a continuous index + to the key does not work if refinement loops of unequal lengths occur for + simulations within a simulation set. + """ - # Check validity if this is not a sub dict if not _sub: verrormsg = 'ccosts must be a dict as returned by JCMsolve.' @@ -264,20 +265,10 @@ def computational_costs_to_flat_dict(ccosts, _sub=False): for key in ccosts: if not key == 'title': val = ccosts[key] + # Sequences longer than 1 occur if a refinement loop was used if is_sequence(val): - if len(val) == 1: - if isinstance(val[0], (string_types, Number)): - converted[key] = val[0] - else: - # Sequences longer than 1 occur if a refinement loop was - # used - for i,v in enumerate(val): - digits = int(np.log10(len(val)))+1 - strfmt = '_{0:0'+str(digits)+'d}' - # We do not allow nested sequences - if not is_sequence(v): - if isinstance(v, (string_types, Number)): - converted[key+strfmt.format(i)] = v + if isinstance(val[-1], (string_types, Number)): + converted[key] = val[-1] else: if isinstance(val, (string_types, Number)): converted[key] = val