From db4d1eb1198628839952dd62376f470060b707b9 Mon Sep 17 00:00:00 2001 From: "Ulf.Andrae" Date: Wed, 27 May 2026 06:22:24 +0200 Subject: [PATCH 1/3] Ulf Andrae: Allow reusage of task index file --- tactus/argparse_wrapper.py | 20 ++++++++++++++++++++ tactus/commands_functions.py | 8 +++----- tactus/submission.py | 4 +++- tactus/tasks/discover_task.py | 8 ++++++-- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/tactus/argparse_wrapper.py b/tactus/argparse_wrapper.py index 5d8cd323..fc040314 100644 --- a/tactus/argparse_wrapper.py +++ b/tactus/argparse_wrapper.py @@ -145,6 +145,7 @@ def get_args_parser(program_name=GeneralConstants.PACKAGE_NAME): required=False, default=False, ) + add_generate_tasklist(parser_run) parser_run.set_defaults(run_command=run_task) ########################################## @@ -219,6 +220,7 @@ def get_args_parser(program_name=GeneralConstants.PACKAGE_NAME): add_keep_def_file( parser_case, help_message="Keep suite definition file in case of submission" ) + add_generate_tasklist(parser_case) add_expand_config(parser_case) parser_case.set_defaults(run_command=create_exp) @@ -250,6 +252,7 @@ def get_args_parser(program_name=GeneralConstants.PACKAGE_NAME): help="Suite definition file", default="", ) + add_generate_tasklist(parser_start_suite) add_keep_def_file(parser_start_suite) parser_start_suite.set_defaults(run_command=start_suite) @@ -551,6 +554,23 @@ def add_keep_def_file( ) +def add_generate_tasklist(parser_object): + """Add object args. + + Args: + parser_object (args oject): args object to update + + """ + parser_object.add_argument( + "--generate-tasklist", + "-g", + help="Force generation of tasklist", + action="store_true", + default=False, + required=False, + ) + + def add_expand_config(parser_object): """Add object args. diff --git a/tactus/commands_functions.py b/tactus/commands_functions.py index ea158b8f..34ae1840 100644 --- a/tactus/commands_functions.py +++ b/tactus/commands_functions.py @@ -31,7 +31,7 @@ from .scheduler import EcflowServer from .submission import NoSchedulerSubmission, TaskSettings from .suites.discover_suite import get_suite -from .tasks.discover_task import create_task_index +from .tasks.discover_task import load_task_index from .toolbox import Platform @@ -98,9 +98,6 @@ def run_task(args: RunTaskNamespace, config: ParsedConfig): submission_defs = TaskSettings(config) sub = NoSchedulerSubmission(submission_defs) - if not args.create_only: - create_task_index(config) - sub.submit( task=args.task, config=config, @@ -109,6 +106,7 @@ def run_task(args: RunTaskNamespace, config: ParsedConfig): output=output, troika=args.troika, create_only=args.create_only, + force_tasklist_generation=args.generate_tasklist, ) logger.info("Task {} submitted.", args.task) @@ -303,7 +301,7 @@ def start_suite(args, config): raise SystemExit(f"Copying {temp_troika_config_file} FAILED.") from e logger.info("--- File copying to Ecflow server DONE ---") - create_task_index(config) + load_task_index(config, force=args.generate_tasklist) server.start_suite(suite_name, def_file) logger.info("Done with suite.") diff --git a/tactus/submission.py b/tactus/submission.py index 53d3b205..5e7f85c1 100644 --- a/tactus/submission.py +++ b/tactus/submission.py @@ -418,6 +418,7 @@ def submit( member: Optional[int] = None, troika: Optional[str] = "troika", create_only: Optional[bool] = False, + force_tasklist_generation: Optional[bool] = False, ): """Submit task. @@ -431,12 +432,13 @@ def submit( Defaults to None. troika (str, optional): troika binary. Defaults to "troika". create_only: (bool, optional): Only create the job, do not submit it. + force_tasklist_generation: (bool, optional): Switch for tasklist generation Raises: RuntimeError: Submission failure. """ name = task.lower() - if name not in load_task_index(config): + if name not in load_task_index(config, force=force_tasklist_generation): raise NotImplementedError(f"Task {name} not implemented") troika_config = Platform(config).get_value("troika.config_file") diff --git a/tactus/tasks/discover_task.py b/tactus/tasks/discover_task.py index 69483c6f..d7c6d2a8 100644 --- a/tactus/tasks/discover_task.py +++ b/tactus/tasks/discover_task.py @@ -60,11 +60,12 @@ def _task_index_file(config): return Path(task_index_file_path) / "tasks_index.json" -def load_task_index(config): +def load_task_index(config, force=False): """Load a task index file. Args: config (ConfigParse): Config + force (boolean): Control of forced tasklist generation Returns: known_types(dict): Dict of known tasks, and their location @@ -72,7 +73,7 @@ def load_task_index(config): """ task_index_file = _task_index_file(config) - if os.path.isfile(task_index_file): + if os.path.isfile(task_index_file) and not force: logger.info("Read task index from {}", task_index_file) with open(task_index_file, "r", encoding="utf-8") as infile: known_types = json.load(infile) @@ -132,6 +133,9 @@ def get_task(name, config) -> Task: try: cls = known_types[name.lower()] except KeyError: + logger.info( + "Task {} not found in index, recreate task index file", name.lower() + ) known_types = create_task_index(config) try: cls = known_types[name.lower()] From 7b890adae51b0d11c7e1ef8ee43f13870f059cb9 Mon Sep 17 00:00:00 2001 From: "Ulf.Andrae" Date: Wed, 3 Jun 2026 07:21:50 +0200 Subject: [PATCH 2/3] Ulf Andrae: Switch logics --- tactus/argparse_wrapper.py | 9 +++++---- tactus/commands_functions.py | 4 ++-- tactus/tasks/discover_task.py | 4 +--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tactus/argparse_wrapper.py b/tactus/argparse_wrapper.py index c736a18f..be7522a6 100644 --- a/tactus/argparse_wrapper.py +++ b/tactus/argparse_wrapper.py @@ -638,12 +638,13 @@ def add_generate_tasklist(parser_object): """ parser_object.add_argument( - "--generate-tasklist", + "--generate-tasklist-off", "-g", - help="Force generation of tasklist", - action="store_true", - default=False, + help="Do not force generation of tasklist", + action="store_false", + default=True, required=False, + dest="force_generation", ) diff --git a/tactus/commands_functions.py b/tactus/commands_functions.py index 080bbd00..30b09896 100644 --- a/tactus/commands_functions.py +++ b/tactus/commands_functions.py @@ -106,7 +106,7 @@ def run_task(args: RunTaskNamespace, config: ParsedConfig): output=output, troika=args.troika, create_only=args.create_only, - force_tasklist_generation=args.generate_tasklist, + force_tasklist_generation=args.force_generation, ) logger.info("Task {} submitted.", args.task) @@ -301,7 +301,7 @@ def start_suite(args, config): raise SystemExit(f"Copying {temp_troika_config_file} FAILED.") from e logger.info("--- File copying to Ecflow server DONE ---") - load_task_index(config, force=args.generate_tasklist) + load_task_index(config, force=args.force_generation) server.start_suite(suite_name, def_file) logger.info("Done with suite.") diff --git a/tactus/tasks/discover_task.py b/tactus/tasks/discover_task.py index d7c6d2a8..43e5b5b7 100644 --- a/tactus/tasks/discover_task.py +++ b/tactus/tasks/discover_task.py @@ -133,9 +133,7 @@ def get_task(name, config) -> Task: try: cls = known_types[name.lower()] except KeyError: - logger.info( - "Task {} not found in index, recreate task index file", name.lower() - ) + logger.info("Task {} not found in index, recreate task index file", name.lower()) known_types = create_task_index(config) try: cls = known_types[name.lower()] From 7913166fa90ae1cb5cf9f79d3e7c27e55e4c321c Mon Sep 17 00:00:00 2001 From: Ulf Andrae Date: Wed, 3 Jun 2026 05:31:00 +0000 Subject: [PATCH 3/3] Ulf Andrae: Fix path def --- .../config_files/config_file_schemas/main_config_schema.json | 5 +++++ tactus/tasks/discover_task.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tactus/data/config_files/config_file_schemas/main_config_schema.json b/tactus/data/config_files/config_file_schemas/main_config_schema.json index 2511e1d3..9e6576b2 100644 --- a/tactus/data/config_files/config_file_schemas/main_config_schema.json +++ b/tactus/data/config_files/config_file_schemas/main_config_schema.json @@ -611,6 +611,11 @@ "description": "Path to the forecast json config file", "default": "@CYCLE@/forecast.json" }, + "task_index_file_path": { + "type": "string", + "description": "Path for task index file", + "default": "@CASEDIR@" + }, "c903_input_definition": { "type": "string", "description": "Path to the c903 json config file", diff --git a/tactus/tasks/discover_task.py b/tactus/tasks/discover_task.py index 43e5b5b7..7d62ac0e 100644 --- a/tactus/tasks/discover_task.py +++ b/tactus/tasks/discover_task.py @@ -56,7 +56,7 @@ def _task_index_file(config): task_index_file (str): Full path to task index_file """ - task_index_file_path = Platform(config).get_system_value("casedir") + task_index_file_path = Platform(config).get_system_value("task_index_file_path") return Path(task_index_file_path) / "tasks_index.json"